def msgpgs(self, ms, psize, tstart, fileMightBeRegistered=False): """ create a progress message from the given info """ pgs = ms.processedSize size = ms.fileSize tnow = time.time() if (pgs is None): pgs = 0 if size == 0: percent = 100 else: percent = int(pgs * 100 / size) tdif = tnow - tstart if tdif == 0: speed = 0 else: speed = (psize + pgs) / (1024*1024*tdif) if speed == 0: speed = 0 eta = " --:--" else: eta = int((self.totalsize - psize - pgs) / (1024*1024*speed)) speed = int(speed) if eta != 0: eta = " {:02d}:{:02d}".format(int(eta/60), eta % 60) else: eta = " 00:00" return "{}{} {:3d}MB/s{}".format("* " if fileMightBeRegistered else "", progressbar(percent,size=21), speed, eta)
def _get_annotations(generator): """ Get the ground truth annotations from the generator. The result is a list of lists such that the size is: all_detections[num_images][num_classes] = annotations[num_detections, 5] # Arguments generator : The generator used to retrieve ground truth annotations. # Returns A list of lists containing the annotations for each image in the generator. """ all_annotations = [[None for i in range(generator.num_classes())] for j in range(generator.size())] for i in progressbar.progressbar(range(generator.size()), prefix='Parsing annotations: '): # load the annotations annotations = generator.load_annotations(i) # copy detections to all_annotations for label in range(generator.num_classes()): if not generator.has_label(label): continue all_annotations[i][label] = annotations['bboxes'][annotations['labels'] == label, :].copy() return all_annotations
def _get_detections(generator, model, score_threshold=0.05, max_detections=100, save_path=None): """ Get the detections from the model using the generator. The result is a list of lists such that the size is: all_detections[num_images][num_classes] = detections[num_detections, 4 + num_classes] # Arguments generator : The generator used to run images through the model. model : The model to run on the images. score_threshold : The score confidence threshold to use. max_detections : The maximum number of detections to use per image. save_path : The path to save the images with visualized detections to. # Returns A list of lists containing the detections for each image in the generator. """ all_detections = [[None for i in range(generator.num_classes()) if generator.has_label(i)] for j in range(generator.size())] for i in progressbar.progressbar(range(generator.size()), prefix='Running network: '): raw_image = generator.load_image(i) image = generator.preprocess_image(raw_image.copy()) image, scale = generator.resize_image(image) if keras.backend.image_data_format() == 'channels_first': image = image.transpose((2, 0, 1)) # run network boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0))[:3] # correct boxes for image scale boxes /= scale # select indices which have a score above the threshold indices = np.where(scores[0, :] > score_threshold)[0] # select those scores scores = scores[0][indices] # find the order with which to sort the scores scores_sort = np.argsort(-scores)[:max_detections] # select detections image_boxes = boxes[0, indices[scores_sort], :] image_scores = scores[scores_sort] image_labels = labels[0, indices[scores_sort]] image_detections = np.concatenate([image_boxes, np.expand_dims(image_scores, axis=1), np.expand_dims(image_labels, axis=1)], axis=1) if save_path is not None: draw_annotations(raw_image, generator.load_annotations(i), label_to_name=generator.label_to_name) draw_detections(raw_image, image_boxes, image_scores, image_labels, label_to_name=generator.label_to_name) cv2.imwrite(os.path.join(save_path, '{}.png'.format(i)), raw_image) # copy detections to all_detections for label in range(generator.num_classes()): if not generator.has_label(label): continue all_detections[i][label] = image_detections[image_detections[:, -1] == label, :-1] return all_detections
def lint_files(filenames, pass_classes, clang_args): def strip_dot_slash(d): if d.filename.startswith('./'): d.filename = d.filename[2:] return d def interesting_file(d): if os.path.isabs(d.filename): return False if d.filename.startswith('opt'): return False return True def mk_task(filename): return (filename, pass_classes, clang_args) diags = [] tasks = (mk_task(f) for f in filenames) if PARALLEL: # TODO: accept -jN and/or read process count from config pool = Pool(processes=4) try: for r in progressbar(pool.imap(worker, tasks), length=len(filenames)): diags += r except KeyboardInterrupt: pool.terminate() return [] else: for r in progressbar(imap(worker, tasks), length=len(filenames)): diags += r diags = map(strip_dot_slash, diags) diags = filter(interesting_file, diags) diags = sorted(diags, key=lambda d: d.line_number) diags = sorted(diags, key=lambda d: d.filename) return unique(diags)
def segment(content): global num_of_line pb = progressbar(num_of_line) ctr = 0 contentlist = list() for item in content: ctr += 1 pb.progress(ctr) seg_list = jieba.cut(item) contentlist.append(','.join(seg_list)) f = open("seged_content.txt", 'wb') for sentence in contentlist: f.write(sentence.encode("utf-8")) f.close()
def evaluate_analogy(args, token_embedding, dataset, analogy_function='ThreeCosMul'): """Evaluation on analogy task.""" # Open vocabulary: Use all known words if args.analogy_max_vocab: counter = nlp.data.Counter(token_embedding.idx_to_token[:args.analogy_max_vocab]) else: counter = nlp.data.Counter(token_embedding.idx_to_token) vocab = nlp.vocab.Vocab(counter) vocab.set_embedding(token_embedding) if args.ignore_oov: initial_length = len(dataset) dataset = [ d for d in dataset if d[0] in vocab and d[1] in vocab and d[2] in vocab and d[3] in vocab ] num_dropped = initial_length - len(dataset) if num_dropped: logging.warning('Dropped %s pairs from %s as the were OOV.', num_dropped, dataset.__class__.__name__) dataset_coded = [[vocab[d[0]], vocab[d[1]], vocab[d[2]], vocab[d[3]]] for d in dataset] dataset_coded_batched = mx.gluon.data.DataLoader( dataset_coded, batch_size=args.batch_size) exclude_question_words = not args.analogy_dont_exclude_question_words evaluator = nlp.embedding.evaluation.WordEmbeddingAnalogy( idx_to_vec=vocab.embedding.idx_to_vec, exclude_question_words=exclude_question_words, analogy_function=analogy_function) context = get_context(args) evaluator.initialize(ctx=context) if not args.dont_hybridize: evaluator.hybridize() acc = mx.metric.Accuracy() if progressbar is not None: dataset_coded_batched = progressbar.progressbar(dataset_coded_batched) for batch in dataset_coded_batched: batch = batch.as_in_context(context) words1, words2, words3, words4 = (batch[:, 0], batch[:, 1], batch[:, 2], batch[:, 3]) pred_idxs = evaluator(words1, words2, words3) acc.update(pred_idxs[:, 0], words4.astype(np.float32)) logging.info('Accuracy on %s: %s', dataset.__class__.__name__, acc.get()[1]) return acc.get()[1], len(dataset)
def filter_template(self, func, s=""): pb = progressbar(len(self.in_q), ".", s) ctr = 0 while True: if self.in_q: ctr += 1 pb.progress(ctr) job = self.in_q.pop() func(job) else: break print len(self.out_q) self.in_q = self.out_q self.out_q = list() return
def filter_stopwords(self): stopwords = self.init_stopwords() pb = progressbar(len(self.in_q), ".", "remove stopwords") ctr = 0 while True: if self.in_q: ctr += 1 pb.progress(ctr) job = self.in_q.pop().split(',') job = filter(lambda x: x not in stopwords, job) self.out_q.append(' '.join(job)) else: break self.in_q = self.out_q self.out_q = list() return
def optimize(args, nmax=2500000): target_dist, m_min, m_max, P, veff_max = args f_new = np.zeros(1000, dtype=[('m', np.float64), ('veff', np.float64), ('v', np.float64), ('x', np.float64), ('t', np.float64)]) f_new['m'] = np.linspace(m_min, m_max, len(f_new)) params_new = np.zeros(5, dtype=[('m', np.float64), ('veff', np.float64)]) params_new['m'] = np.linspace(m_min, m_max, len(params_new)) params_new['veff'] = 0.1 calculate(f_new, P, dist=target_dist, veff_max=veff_max, params=params_new) score_best = get_score(f_new, dist=target_dist) f_orig = f_new.copy() last_i = 0 for i in progressbar(range(nmax)): g_params = step(params_new, veff_max=veff_max) g_g = f_new.copy() mis_t, mis_x = calculate(g_g, P, dist=target_dist, params=g_params, veff_max=veff_max) score = get_score(g_g, dist=target_dist) if (i+1) % (nmax / 20) == 0: for j in range(2): g_ind = np.random.randint(0, len(params_new)//2-1) + j * len(params_new)//2 obj = params_new.copy()[0] obj['m'] = (params_new['m'][g_ind] + params_new['m'][g_ind+1]) / 2 obj['veff'] = (params_new['veff'][g_ind] + params_new['veff'][g_ind + 1]) / 2 params_new = np.insert(params_new, g_ind+1, obj) g_params = params_new.copy() if score < score_best: if (score < 800) and (i-last_i) > 1000 and False: last_i = i plt.plot(g_g['m'] / 1000, g_g['veff'] / 1000, label='optimized') plt.show() plt.plot(g_g['m'] / 1000, g_g['veff'] / 1000, label='optimized') score_best = score f_new = g_g params_new = g_params return f_new, f_orig, score_best, mis_t
def render_to_directory(dirname, title, files, category_names): try: os.mkdir(dirname) except OSError: pass index_template = load_template('report_index.jinja') single_file_template = load_template('report_single_file.jinja') with open(os.path.join(dirname, 'index.html'), 'w') as fo: fo.write(index_template.render(files=files, diagnostic_categories=category_names, title=title).encode('utf8')) for f in progressbar(files): for dname, ds in f['diagnostic_groups'].iteritems(): path = os.path.join(dirname, f['name'].replace('/', '_')) path += '_' + dname + '.html' text = single_file_template.render(filename=f['name'], diagnostics=ds) with open(path, 'w') as fo: fo.write(text.encode('utf8'))
def process(content): global num_of_line pb = progressbar(num_of_line) ctr = 0 worddict = dict() wordlist = list() for item in content: ctr += 1 pb.progress(ctr) seg_list = jieba.cut(item) for word in seg_list: if worddict.has_key(word): worddict[word] += 1 else: worddict[word] = 1 print "total words:", len(worddict) temp = list() for k, v in worddict.iteritems(): temp.append([k, v]) wordlist = sorted(temp, key=lambda x: x[1], reverse=True) f = open("terms.txt", 'wb') for word in wordlist: f.write(word[0] + '\t' + str(word[1]) + '\n') f.close()
times.SetName('Time') vtkilagr = vtk.vtkIdTypeArray() vtkilagr.SetNumberOfComponents(1) vtkilagr.SetNumberOfTuples(pts.GetNumberOfPoints()) vtkilagr.SetName('ILAGR') vtkexist = vtk.vtkShortArray() vtkexist.SetNumberOfComponents(1) vtkexist.SetNumberOfTuples(pts.GetNumberOfPoints()) vtkexist.SetName('EXIST') #verts = vtk.vtkCellArray() for j in progressbar.progressbar(range(N)): pts.SetPoint(j, (x.iloc[j],y.iloc[j],z.iloc[j])) types.SetTuple1(j, tp.iloc[j]) times.SetTuple1(j, T.iloc[j]) vtkilagr.SetTuple1(j, ilagr.iloc[j]) vtkexist.SetTuple1(j, exist.iloc[j]) # verts.InsertNextCell(1) # verts.InsertCellPoint(j) #create connectivity #get unique IDs uIds = pd.unique(ilagr)
def preciseNameSearch(*names): nameData = [] errName = [] # a list contain names that fail to find in preciseSearch combinedNameData = [] counter = 1 k = 0 totalRecordNum = len(names[0]) for k in progressbar.progressbar(range(len(names[0])), redirect_stdout=True): name = names[0][k] processedName = URI_escape(name) print("Processing ", counter, " / ", totalRecordNum, " of records...", end="") nameURL = 'https://id.loc.gov/authorities/names/label/' + processedName try: nameResponse = requests.head(nameURL) except: print("Fail to request: ", name) errName.append(name) if nameResponse.status_code == 302: nameData.append(name) nameData.append(name) nameData.append(nameResponse.headers['X-Uri']) nameData.append(nameResponse.headers['X-Preflabel']) else: if nameResponse.status_code == 301: print( "\nHTTP 301- Request link has been moved permanently....", end="") elif nameResponse.status_code == 404: print("\nHTTP 404- Request link page doesn't exist....", end="") else: print("\n HTTP", subjectResponse.status_code, "- Not a valid response for scraping....", end="") #TODO: name not appended, no err file out nameData.append(name) nameData.append("null") nameData.append("null") nameData.append("null") combinedNameData.append(nameData) print("Done!") if (len(errName) > 0): errName.insert("Name") errorOut = open("Err.csv", 'w', encoding='utf-8', newline='') simpleCSV.writeCSV(["Name with no result"], errorOut) SimpleCSV.writeCSV(errName, errorOut) print("Name that cause error is put into Err.csv") counter = counter + 1 k = k + 1 nameData = [] #add header for output file combinedNameData.insert(0, ["Raw Name", "Correct Name", "LC_URI", "LC_Label"]) return combinedNameData
def _get_detections(generator, model, score_threshold=0.05, max_detections=100, save_path=None, save_path_image=None): """ Get the detections from the model using the generator. The result is a list of lists such that the size is: all_detections[num_images][num_classes] = detections[num_detections, 4 + num_classes] # Arguments generator : The generator used to run images through the model. model : The model to run on the images. score_threshold : The score confidence threshold to use. max_detections : The maximum number of detections to use per image. save_path : The path to save the images with visualized detections to. # Returns A list of lists containing the detections for each image in the generator. """ all_detections = [[ None for i in range(generator.num_classes()) if generator.has_label(i) ] for j in range(generator.size())] all_inferences = [None for i in range(generator.size())] result = {'path': [], 'x1': [], 'y1': [], 'x2': [], 'y2': [], 'class': []} for i in progressbar.progressbar(range(generator.size()), prefix='Running network: '): raw_image = generator.load_image(i) image = generator.preprocess_image(raw_image.copy()) image, scale = generator.resize_image(image) image_path = generator.image_path(i) if keras.backend.image_data_format() == 'channels_first': image = image.transpose((2, 0, 1)) # run network start = time.time() boxes, scores, labels = model.predict_on_batch( np.expand_dims(image, axis=0))[:3] inference_time = time.time() - start # correct boxes for image scale boxes /= scale # select indices which have a score above the threshold indices = np.where(scores[0, :] > score_threshold)[0] # select those scores scores = scores[0][indices] # find the order with which to sort the scores scores_sort = np.argsort(-scores)[:max_detections] # select detections image_boxes = boxes[0, indices[scores_sort], :] image_scores = scores[scores_sort] image_labels = labels[0, indices[scores_sort]] image_detections = np.concatenate([ image_boxes, np.expand_dims(image_scores, axis=1), np.expand_dims(image_labels, axis=1) ], axis=1) if save_path is not None: selection = np.where(image_scores > score_threshold)[0] for j in selection: b = np.array(image_boxes[j, :]).astype(int) result['path'].append(image_path) result['x1'].append(b[0]) result['y1'].append(b[1]) result['x2'].append(b[2]) result['y2'].append(b[3]) result['class'].append(image_labels[j]) if save_path_image is not None: draw_annotations(raw_image, generator.load_annotations(i), label_to_name=generator.label_to_name) draw_detections(raw_image, image_boxes, image_scores, image_labels, label_to_name=generator.label_to_name, score_threshold=score_threshold) cv2.imwrite(os.path.join(save_path_image, '{}.png'.format(i)), raw_image) # copy detections to all_detections for label in range(generator.num_classes()): if not generator.has_label(label): continue all_detections[i][label] = image_detections[ image_detections[:, -1] == label, :-1] all_inferences[i] = inference_time if save_path: df = pd.DataFrame(result) df.to_csv(save_path, sep=',', index=False, header=None) return all_detections, all_inferences
def connect_knn(args, ws): if args.no_wait: return trial_str = _trial_str(ws.current_trial) algoprefix = 'pairwise_knn-' for puzzle_fn, puzzle_name in ws.test_puzzle_generator(args.puzzle_name): ITE = matio.load(_ibte_fn( ws, puzzle_name, ws.current_trial))['INTER_BLOOMING_TREE_EDGES'] pds_fn = _puzzle_pds(ws, puzzle_name, ws.current_trial) d = matio.load(pds_fn) QF = d['QF'] QB = d['QB'] """ We need to be aware of one important differents between bloom-from_*.npz files and the PDS file. Some blooming tree may be empty. Hence the Bloom No. (denoted by the number in the file name) may not be aligned to the index in the PDS file. However, the inter_blooming_tree_edges (ITE) uses bloom no. rather than the PDS index. Thus the translation is needed. """ BLOOM_NO_TO_INDEX = d['BLOOM_NO_TO_INDEX'] INDEX_TO_BLOOM_NO = d['INDEX_TO_BLOOM_NO'] import networkx as nx # Forest level path G = nx.Graph() G.add_nodes_from([i for i in range(len(BLOOM_NO_TO_INDEX))] + [VIRTUAL_OPEN_SPACE_NODE]) G.add_edges_from(ITE[:, [0, 2]]) openset = [] virtual_edges = [] """ Connect OpenSet trees to VIRTUAL_OPEN_SPACE_NODE """ for tree_index, tree_base in enumerate(QB): _, tree_end = _extract_bound(QB, QF.shape[0], tree_index) """ Translate back to bloom no. to be compatitable with ITE """ tree_no = INDEX_TO_BLOOM_NO[tree_index] # print(tree_base) # print(tree_end) for flag in QF[tree_base:tree_end]: if flag & OPENSPACE_FLAG: openset.append(tree_no) virtual_edges.append((tree_no, VIRTUAL_OPEN_SPACE_NODE)) break util.log("OpenSet {}".format(openset)) G.add_edges_from(virtual_edges) try: ids = nx.shortest_path(G, 0, VIRTUAL_OPEN_SPACE_NODE) util.log('Forest-level shortest path {}'.format(ids)) except nx.exception.NetworkXNoPath: util.warn( f'[connect_knn] Cannot find path for puzzle {puzzle_name}') continue util.ack( '[solve1][connect_knn] forest level path (bloom no.) {}'.format( ids)) ids = [BLOOM_NO_TO_INDEX[i] for i in ids[:-1]] ids.append(VIRTUAL_OPEN_SPACE_NODE) util.ack( '[solve1][connect_knn] forest level path (PDS index) {}'.format( ids)) """ We use bloom no. in forest level path, because ITE uses bloom no. In tree level path we uses pds index instead, because we tree level data mainly comes from the PDS file. """ ITE_meta = {} def insert_ite(from_fi, from_vi, to_fi, to_vi): ''' if from_fi == 0 and to_fi == 78: util.log('ite {}'.format(ite)) ''' if from_fi not in ITE_meta: ITE_meta[from_fi] = {} if to_fi not in ITE_meta[from_fi]: ITE_meta[from_fi][to_fi] = [(from_vi, to_vi)] else: ITE_meta[from_fi][to_fi].append((from_vi, to_vi)) for index, ite in progressbar(enumerate(ITE)): from_fi, from_vi, to_fi, to_vi = ite from_fi = BLOOM_NO_TO_INDEX[from_fi] to_fi = BLOOM_NO_TO_INDEX[to_fi] insert_ite(from_fi, from_vi, to_fi, to_vi) insert_ite(to_fi, to_vi, from_fi, from_vi) # from_fi, to_fi = 0, 82 # util.log(f'ITE_meta[{from_fi}][{to_fi}] {ITE_meta[from_fi][to_fi]}') Q = d['Q'] QE = d['QE'] QEB = d['QEB'] ompl_q = [] bloom0_fn = ws.local_ws( _rel_bloom_scratch(ws, puzzle_name, ws.current_trial), f'bloom-from_0.npz') prev_fi = 0 prev_vi = matio.load(bloom0_fn)['IS_INDICES'][0] try: for from_fi, to_fi in zip(ids, ids[1:]): if to_fi != VIRTUAL_OPEN_SPACE_NODE: from_vi, to_vi = ITE_meta[from_fi][to_fi][0] else: from_fi = prev_fi from_vi = VIRTUAL_OPEN_SPACE_NODE # if to_vi == VIRTUAL_OPEN_SPACE_NODE: # ids = ids[:-1] ids = tree_level_path(Q, QB, QE, QEB, QF, prev_fi, prev_vi, from_fi, from_vi) q_from = QB[from_fi] local_ids = ids - q_from util.log(f"Tree {from_fi} IDS {ids}, Local IDS {local_ids}") qs = Q[ids, :] util.log(f"Shape {qs.shape}") """ bloom_fn = ws.local_ws(_rel_bloom_scratch(ws, puzzle_name, ws.current_trial), f'bloom-from_{from_fi}.npz') localQ = matio.load(bloom_fn)['BLOOM'] for gi,li in zip(ids, local_ids): dist = np.linalg.norm(Q[gi] - localQ[li]) assert dist < 1e-3, f'{Q[gi]} != {localQ[li]} gi {gi} li {li}' """ ompl_q.append(qs) # if to_fi != VIRTUAL_OPEN_SPACE_NODE: # gvi = QB[to_fi] + to_vi # ompl_q.append(Q[gvi:gvi+1,:]) prev_fi = to_fi prev_vi = to_vi # ompl_q.append(tree_level_path(Q, QB, QE, QEB, QF, prev_fi, prev_vi, prev_fi, VIRTUAL_OPEN_SPACE_NODE)) except nx.exception.NetworkXNoPath: assert False, "Should not happen. Found forest-level path but no tree-level path." continue ompl_q = util.safe_concatente(ompl_q, axis=0) rel_scratch_dir = os.path.join(util.SOLVER_SCRATCH, puzzle_name, trial_str) path_out = ws.local_ws(rel_scratch_dir, algoprefix + 'path.txt') matio.savetxt(path_out, ompl_q) util.ack("Saving OMPL solution of {} to {}".format( puzzle_name, path_out)) uw = util.create_unit_world(puzzle_fn) unit_q = uw.translate_ompl_to_unit(ompl_q) sol_out = ws.solution_file(puzzle_name, type_name=algoprefix + 'unit') matio.savetxt(sol_out, unit_q) util.ack("Saving UNIT solution of {} to {}".format( puzzle_name, sol_out))
def temp_tuning(image: np.array, T_MAX, N, initial: np.array = None): if initial is None: initial = hiprmc.random_initial(image) T = T_MAX # T = 0 simulated_image = initial.copy() original = simulated_image F_image = hiprmc.fourier_transform(image) F_old = hiprmc.fourier_transform(simulated_image) chi_old = hiprmc.chi_square(F_old, F_image) error, count, accept_rate, temperature = [], [], [], [] t_step = T_MAX / 100.0 for t in progressbar.progressbar(range(0, 100)): move_count = 0.0 acceptance = 0.0 T = T - t_step for iter in range(0, np.count_nonzero(simulated_image == 1)): move_count += 1 x_old = np.random.randint(0, image.shape[0]) y_old = np.random.randint(0, image.shape[1]) while simulated_image[x_old][y_old] != 1: x_old = np.random.randint(0, image.shape[0]) y_old = np.random.randint(0, image.shape[1]) x_new = np.random.randint(0, image.shape[0]) y_new = np.random.randint(0, image.shape[1]) while np.sqrt((y_new - y_old) ** 2 + (x_new - x_old) ** 2) > 5: x_new = np.random.randint(0, image.shape[0]) y_new = np.random.randint(0, image.shape[1]) old_point = simulated_image[x_old][y_old] new_point = simulated_image[x_new][y_new] while new_point == old_point: x_new = np.random.randint(0, image.shape[0]) y_new = np.random.randint(0, image.shape[1]) new_point = simulated_image[x_new][y_new] simulated_image[x_old][y_old] = new_point simulated_image[x_new][y_new] = old_point old_array = np.zeros_like(image) old_array[x_old][y_old] = 1 new_array = np.zeros_like(image) new_array[x_new][y_new] = 1 F_new = F_old - hiprmc.fourier_transform(old_array) + hiprmc.fourier_transform(new_array) chi_new = hiprmc.chi_square(F_new, F_image) delta_chi = chi_new - chi_old acceptance, chi_old = hiprmc.Metropolis(x_old, y_old, x_new, y_new, old_point, new_point, delta_chi, simulated_image, T, acceptance, chi_old, chi_new, T_MAX, iter, N) F_old = hiprmc.fourier_transform(simulated_image) error.append(chi_old) count.append(t) accept_rate.append(acceptance / move_count) temperature.append(T) # def gaussian(x, a, b): # return a * (1 - np.exp(-(x / b) ** 2)) def sigmoid(x, a, b): return 1.0 / (1.0 + np.exp(-a * (x - b))) fit_params, fit_params_error = optimize.curve_fit(sigmoid, temperature, accept_rate, p0=[0.5, T_MAX / 2]) # f = plt.figure() # f.add_subplot(1, 2, 1) # plt.plot(temperature, accept_rate, 'bo') # plt.plot(temperature, sigmoid(temperature, *fit_params), 'r-') # plt.ylabel('Acceptance Rate') # plt.ylim([0, 1]) # plt.xlabel('Temperature') # f.add_subplot(1, 2, 2) # plt.plot(count, error, 'k-') # plt.ylabel('Chi-Squared') # plt.xlabel('Monte Carlo Iteration') # plt.tight_layout() # plt.show() # temp_tstar = abs(fit_params[1]) * np.sqrt(-np.log(1 - 0.333333 / fit_params[0])) temp_tstar = fit_params[1] - np.log(2.3333) / fit_params[0] t_min = max(0.001, temp_tstar - 0.2) t_max = min(T_MAX, temp_tstar + 0.2) t_step = (t_max - t_min) / 1000 T = t_max simulated_image = original print(t_min, t_max) temperature2, accept_rate2 = [], [] for t in progressbar.progressbar(range(0, 1000)): move_count = 0.0 acceptance = 0.0 for iter in range(0, np.count_nonzero(simulated_image == 1)): move_count += 1 x_old = np.random.randint(0, image.shape[0]) y_old = np.random.randint(0, image.shape[1]) while simulated_image[x_old][y_old] != 1: x_old = np.random.randint(0, image.shape[0]) y_old = np.random.randint(0, image.shape[1]) x_new = np.random.randint(0, image.shape[0]) y_new = np.random.randint(0, image.shape[1]) while np.sqrt((y_new - y_old) ** 2 + (x_new - x_old) ** 2) > 15: x_new = np.random.randint(0, image.shape[0]) y_new = np.random.randint(0, image.shape[1]) old_point = simulated_image[x_old][y_old] new_point = simulated_image[x_new][y_new] while new_point == old_point: x_new = np.random.randint(0, image.shape[0]) y_new = np.random.randint(0, image.shape[1]) new_point = simulated_image[x_new][y_new] simulated_image[x_old][y_old] = new_point simulated_image[x_new][y_new] = old_point old_array = np.zeros_like(image) old_array[x_old][y_old] = 1 new_array = np.zeros_like(image) new_array[x_new][y_new] = 1 F_new = F_old - hiprmc.fourier_transform(old_array) + hiprmc.fourier_transform(new_array) chi_new = hiprmc.chi_square(F_new, F_image) delta_chi = chi_new - chi_old acceptance, chi_old = hiprmc.Metropolis(x_old, y_old, x_new, y_new, old_point, new_point, delta_chi, simulated_image, T, acceptance, chi_old, chi_new, T_MAX, iter, N) F_old = hiprmc.fourier_transform(simulated_image) accept_rate2.append(acceptance / move_count) temperature2.append(T) T = T - t_step fit_params, fit_params_error = optimize.curve_fit(sigmoid, temperature2, accept_rate2, p0=[0.5, temp_tstar]) # temp_tstar = abs(fit_params[1]) * np.sqrt(-np.log(1 - 0.333333 / fit_params[0])) temp_tstar = fit_params[1] - np.log(2) / fit_params[0] t_min = min(0.001, temp_tstar - 0.2) t_max = min(T_MAX, temp_tstar + 0.2) # # f = plt.figure() # f.add_subplot(1, 2, 1) # plt.plot(temperature2, accept_rate2, 'bo') # plt.plot(temperature2, sigmoid(temperature2, *fit_params), 'r-') # plt.ylabel('Acceptance Rate') # plt.ylim([0,1]) # plt.xlabel('Temperature') # f.add_subplot(1, 2, 2) # plt.plot(count, error, 'k-') # plt.ylabel('Chi-Squared') # plt.xlabel('Monte Carlo Iteration') # plt.tight_layout() # plt.show() # # simulated_image = original # f = plt.figure() # f.add_subplot(1, 5, 1) # plt.axis('off') # plt.title("original image") # f.add_subplot(1, 5, 2) # plt.axis('off') # plt.title("FFT of Image") # f.add_subplot(1, 5, 3) # plt.imshow(initial, cmap="Greys", origin="lower") # plt.title("random start") # plt.axis('off') # f.add_subplot(1, 5, 4) # plt.imshow(simulated_image, cmap="Greys", origin="lower") # plt.title("updated") # plt.axis('off') # f.add_subplot(1, 5, 5) # plt.imshow(np.log10(abs(hiprmc.fourier_transform(simulated_image))), origin='lower') # plt.axis('off') # plt.title("FFT of final state") # plt.show() return simulated_image, t_max, t_min
def subset(self): print('Processing request:') for i in progressbar.progressbar(range(100)): time.sleep(0.08) print('Request processing complete.')
def test(net, dataloader, criterion=None, evaluate=True, predict=True, device=torch.device('cuda:0')): ''' 使用训练好的net进行预测或评价 args: net,训练好的RetinaNet对象; dataloader,需要进行预测或评价的数据集的dataloader; criterion,计算损失的函数,如果是None则不计算loss; evaluate,如果是True则进行模型的评价(即计算mAP),如果 是False,则只进行预测或计算loss; predict,是否进行预测,如果True则会返回预测框的类别和坐标,如果False, 则不会返回; returns: all_labels_pred, all_markers_pred,如果predict=True,则返回预测的每张图 片的预测框的标签和loc; losses, cls_losses, loc_losses,如果criterion不是None,则得到其3个loss; APs, mAP_score,如果evaluate=True,则返回每一类的AP值和其mAP; ''' y_encoder = dataloader.dataset.y_encoder y_encoder_mode = dataloader.dataset.out assert (evaluate and y_encoder_mode in ['obj', 'all']) or not evaluate assert (criterion is not None and y_encoder_mode in ['all', 'encode']) or \ criterion is None print('Testing ...') results = [] with torch.no_grad(): losses = 0. all_labels_pred = [] # 预测的labels, all_scores_pred = [] all_markers_pred = [] all_labels_true = [] all_markers_true = [] for batch in pb.progressbar(dataloader): imgs = batch[0].to(device) if y_encoder_mode in ['all', 'obj']: labels_true, locs_true = batch[1:3] if y_encoder_mode in ['all', 'encode']: targets = batch[-1].to(device) preds = net(imgs) if criterion is not None: loss = criterion(preds, targets) losses += loss.item() if predict or evaluate: img_size = list(imgs.shape[-1:-3:-1]) for pred in preds: res_c, res_s, res_l = y_encoder.decode(pred, img_size) all_labels_pred.append(res_c) all_scores_pred.append(res_s) all_markers_pred.append(res_l) if evaluate: all_labels_true.extend(labels_true) all_markers_true.extend(locs_true) if predict: results.append((all_labels_pred, all_markers_pred)) if criterion is not None: losses = losses / len(dataloader.dataset) results.append(losses) if evaluate: # 使用chain将两层的嵌套list变成一层,符合mAP函数的输出要求 APs, mAP_score = mAP( all_labels_true, all_markers_true, all_labels_pred, all_scores_pred, all_markers_pred, ) results.append((APs, mAP_score)) return tuple(results)
'A2', 'A3', 'A4', 'A5', 'A6', 'A7', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7' ] classes_Ra = np.array([0.05, 0.1, 0.2, 0.4, 0.8, 1.6]) # A_classes = ['A2', 'A3', 'A4', 'A5', 'A6', 'A7'] # B_classes = ['B2', 'B3', 'B4', 'B5', 'B6', 'B7'] img_per_class = 2200 img_names = list(imagenames(img_per_class)) nb_classes = 12 nb_channels = 1 test_ratio = 0.1 print("Loading data... ", flush=True) # Loading Images data = np.empty((img_per_class * nb_classes, *img_shape), dtype=np.float) for i in progressbar(range(nb_classes)): class_name = classes[i] for j, img in enumerate(img_names): imgpath = root + class_name + '/' + img data[i * img_per_class + j, ...] = imread(imgpath) data_full = data.reshape((-1, *img_shape_flat)) data_A, data_B = np.split(data_full, 2) data_classes = np.split(data_full, nb_classes) del data # Creating labels labels_full = np.arange(nb_classes) labels_full = np.repeat(labels_full, img_per_class) labels_half = np.arange(nb_classes / 2) labels_merged = np.tile(labels_half, 2)
def transform(self, X: dt.Frame): """ Transform expects only one column that contains a file name :param X: contains file names :return: features created on signals contained in the files """ # First we want to make sure that: # - X contains only 1 column # - The column is text # - The column contains file names from progressbar import progressbar import numba @numba.jit(parallel=True, fastmath=True) def get_rolling_min(seq): window = 30 l = len(seq) - window z = np.empty(l) for i in numba.prange(l): z[i] = np.min(seq[i:i + window]) return z @numba.jit(parallel=True, fastmath=True) def get_rolling_max(seq): window = 30 l = len(seq) - window z = np.empty(l) for i in numba.prange(l): z[i] = np.max(seq[i:i + window]) return z def get_nb_events(file, levels): sig = dt.fread(file).to_numpy()[:, 0] a = get_rolling_min(sig) b = get_rolling_max(sig) z = np.log10(b - a + 1e-10) return [np.sum(z[z > _level]) for _level in levels] if X.shape[1] > 1: return np.zeros(X.shape[0]) if isinstance(X, dt.Frame): # Datatable can select features directly on type if X[:, [str]].shape[1] == 0: return np.zeros(X.shape[0]) files = X[:, [str]].to_numpy()[:, 0] else: if X[X.columns[0]].dtype != "object": return np.zeros(X.shape[0]) files = X[X.columns[0]].values[:, 0] # Now let's go through the files and create features try: # Here we are supposed to use numba so multi processing is not required levels = np.arange(1.0, 1.2, 0.1) ret_df = pd.DataFrame([ get_nb_events(file, levels) # for file in files for file in progressbar(files) ]) except ValueError as e: err_msg = e.args[0] if "file" in err_msg.lower() and "does not exist" in err_msg.lower( ): print("Error in {} : {}".format(self.display_name, err_msg)) return np.zeros(X.shape[0]) # Use pandas instead of dt.Frame(features) # Pending issue #9894 return ret_df.values
def transform(self, X: dt.Frame): """ Transform expects only one column that contains a file name :param X: contains file names :return: features created on signals contained in the files """ from progressbar import progressbar # First we want to make sure that: # - X contains only 1 column # - The column is text # - The column contains file names # Make sure we have 1 column if X.shape[1] > 1: return np.zeros(X.shape[0]) # Extract file paths if isinstance(X, dt.Frame): # Datatable can select features directly on type if X[:, [str]].shape[1] == 0: return np.zeros(X.shape[0]) files = X[:, [str]].to_numpy()[:, 0] else: if X[X.columns[0]].dtype != "object": return np.zeros(X.shape[0]) files = X[X.columns[0]].values[:, 0] # Now go through the files and create features try: def processor(out, res): # print(out) # print(res, flush=True) out.append(res[0]) # out[res[0]] = res[1] num_tasks = X.shape[0] pool_to_use = small_job_pool pool = pool_to_use(logger=None, processor=processor, num_tasks=num_tasks) # Features will be a list of dict features = [] for i_f, file in enumerate(progressbar(files)): # Create full path full_path = file # Send to pool args = (i_f, full_path, self._mfcc_size) kwargs = {} pool.submit_tryget(None, get_features, args=args, kwargs=kwargs, out=features) pool.finish() features_df = pd.DataFrame(features) self._output_feature_names = list(features_df.columns) self._feature_desc = list(features_df.columns) return features_df except ValueError as e: err_msg = e.args[0] if "file" in err_msg.lower() and "does not exist" in err_msg.lower( ): print("Error in {} : {}".format(self.display_name, err_msg)) return np.zeros(X.shape[0])
Plin = 10**(np.asarray(dset['input/PdB'][...]/10)) try: obj = dset.create_dataset('SCA', shape = dset['objval'].shape, fillvalue = np.nan, dtype = dset['objval'].dtype) popt = dset.create_dataset('SCA_xopt', shape = dset['objval'].shape + (4,), fillvalue = np.nan, dtype = dset['objval'].dtype) obj2 = dset.create_dataset('SCAmax', shape = dset['objval'].shape, fillvalue = np.nan, dtype = dset['objval'].dtype) popt2 = dset.create_dataset('SCAmax_xopt', shape = dset['objval'].shape + (4,), fillvalue = np.nan, dtype = dset['objval'].dtype) except RuntimeError: obj = dset['SCA'] popt = dset['SCA_xopt'] obj2 = dset['SCAmax'] popt2 = dset['SCAmax_xopt'] pt = None for cidx, pidx in pb.progressbar(it.product(range(11907,20000), range(obj.shape[1])), widget = pb.ETA, max_value = (20000-11907)*obj.shape[1]): h = np.asarray(dset['input/channel_to_noise_matched'][cidx], dtype = float) p = Plin[pidx] if pidx == 0: pt = None if pt is not None: o1,p1 = SCA(h, mu, Pc, Pmax = p, pt = pt) else: o1 = -np.inf o2,p2 = SCA(h, mu, Pc, Pmax = p, pt = np.full(4, p)) obj2[cidx,pidx] = o2 popt2[cidx,pidx,:] = p2
import cv2 import sys from os.path import join from os import walk, makedirs from progressbar import progressbar srcDir = sys.argv[1] dstDir = sys.argv[2] dirs = [join(root, dir) for root, dirs, files in walk(srcDir) for dir in dirs] for dir in dirs: newDir = join(dstDir, dir.replace(srcDir, "")) makedirs(newDir, exist_ok=True) imgFiles = [ join(root, file) for root, dirs, files in walk(srcDir) for file in files if file.endswith(".png") ] #print(dirs) for imgFile in progressbar(imgFiles): dstImgFile = join(dstDir, imgFile.replace(srcDir, "")) img = cv2.imread(imgFile) rescaledImg = cv2.resize(img, dsize=(1280, 720), interpolation=cv2.INTER_CUBIC) cv2.imwrite(dstImgFile, rescaledImg)
def solve(self, n_iteration=100, save_likelihood=False, save_parameter=False, save_wav=False, save_path="./", interval_save_parameter=30, mic_index=MIC_INDEX): """ Parameters: save_likelihood: boolean save likelihood and lower bound or not save_parameter: boolean save parameter or not save_wav: boolean save intermediate separated signal or not save_path: str directory for saving data interval_save_parameter: int interval of saving parameter """ self.n_iteration = n_iteration self.initialize_PSD() self.initialize_covarianceMatrix() self.make_filename_suffix() log_likelihood_array = [] for it in progressbar(range(self.n_iteration)): self.update() if save_parameter and ((it + 1) % interval_save_parameter == 0) and ((it + 1) != self.n_iteration): self.save_parameter( save_path + "{}-parameter-{}-{}.pic".format( self.method_name, self.filename_suffix, it + 1)) if save_wav and ((it + 1) % interval_save_parameter == 0) and ((it + 1) != self.n_iteration): self.separate_FastWienerFilter(mic_index=MIC_INDEX) self.save_separated_signal( save_path + "{}.wav".format(self.filename_suffix)) if save_likelihood and ((it + 1) % interval_save_parameter == 0) and ((it + 1) != self.n_iteration): log_likelihood_array.append(self.calculate_log_likelihood()) if save_parameter: self.save_parameter(save_path + "{}-parameter-{}.pic".format( self.method_name, self.filename_suffix)) if save_likelihood: log_likelihood_array.append(self.calculate_log_likelihood()) pic.dump( log_likelihood_array, open( save_path + "{}-likelihood-interval={}-{}.pic".format( self.method_name, interval_save_parameter, self.filename_suffix), "wb")) self.separate_FastWienerFilter(mic_index=mic_index) self.save_separated_signal(save_path + "{}.wav".format(self.filename_suffix))
bar = progressbar.ProgressBar(max_value=len(local_time_global_ids)) with mp.Pool(processes = ncpus) as p: for i, _ in enumerate( p.imap_unordered(save_one_file, local_time_global_ids), 1 ): bar.update(i) print('Generating main pvd file') with open(main_filename,'w') as f: f.write('<?xml version="1.0"?>\n') f.write('<VTKFile type="Collection" version="0.1"\n') f.write(' byte_order="LittleEndian"\n') f.write(' compressor="vtkZLibDataCompressor">\n') f.write(' <Collection>\n') for i in progressbar.progressbar(range(len(times))): filename = "pts_{:010d}.vtp".format(i) if isfile(filename): f.write(f'<DataSet timestep="{times[i]}" group="" part="0"\n') f.write(f' file="{filename}"/>\n') f.write(' </Collection>\n') f.write('</VTKFile>\n')
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) # labels file parser.add_argument('labels_file') # en-cn file parser.add_argument('en_cn_file') # directory where contains labelme annotated json files parser.add_argument('in_dir', help='input dir with annotated files') # output directory for dataset parser.add_argument('out_dir', help='output dataset directory') args = parser.parse_args() # build a chinese-english label convert dict (cn2ens, en2cns) = en_cn_dict_build(args.en_cn_file) # regex pattern(used in get image name) pattern = re.compile(r"\d+") # e.g. use time as file name # 1. Create Directories # remove output dir manually when exists if osp.exists(args.out_dir): print('Output directory already exists:', args.out_dir) quit(1) # make voc format directories os.makedirs(args.out_dir) os.makedirs(osp.join(args.out_dir, 'JPEGImages')) os.makedirs(osp.join(args.out_dir, 'Annotations')) os.makedirs(osp.join(args.out_dir, 'AnnotationsVisualization')) print('dataset dir is:', args.out_dir) # 2. Get Label Information # get labels and save it to dataset dir class_names = [] for i, line in enumerate( open(args.labels_file, 'r', encoding='UTF-8').readlines()): class_id = i - 1 # starts with -1 class_name = line.strip() if class_id == -1: assert class_name == '__ignore__' continue elif class_id == 0: assert class_name == '_background_' class_names.append(class_name) else: class_names.append(cn2ens[class_name]) class_names = tuple(class_names) print('class_names:', class_names) out_class_names_file = osp.join( args.out_dir, 'class_names.txt') # save labels in txt for information with open(out_class_names_file, 'w', encoding='UTF-8') as f: f.writelines('\n'.join(class_names)) print('Saved class_names:', out_class_names_file) # 3. Process Every Json File label_file_list = glob.glob(osp.join(args.in_dir, '*.json')) for i in progressbar.progressbar(range(len(label_file_list))): label_file = label_file_list[i] # load json # print('Generating dataset from:', label_file) with open(label_file, 'r', encoding='UTF-8') as f: data = json.load(f) # regex get img name filename = osp.splitext(osp.basename(label_file))[0] base = pattern.findall(filename)[0] # src image file out_img_file = osp.join(args.out_dir, 'JPEGImages', base + '.jpg') # annotation xml file out_xml_file = osp.join(args.out_dir, 'Annotations', base + '.xml') # visualize image file out_viz_file = osp.join(args.out_dir, 'AnnotationsVisualization', base + '_Viz.jpg') # color annotated image file out_colorize_file = osp.join(args.out_dir, 'AnnotationsVisualization', base + '_ColorViz.jpg') # labelme annotated file contains source image data(serialized) # get img data as numpy array imgdata = data['imageData'] img = utils.img_b64_to_arr(imgdata) # also you can read image from img path too # img_file = osp.join(osp.dirname(label_file), data['imagePath']) # img = np.asarray(PIL.Image.open(img_file)) # save the origin image file PIL.Image.fromarray(img).save(out_img_file) # lxml is a wonderful xml tool # generate voc format annotation file maker = lxml.builder.ElementMaker() xml = maker.annotation( maker.folder(""), # folder name maker.filename(base + '.jpg'), # img path maker.source( # img source, doesn't matter maker.database(""), maker.annotation(""), maker.image(""), ), maker.size( # image size(height, width and channel) maker.height(str(img.shape[0])), maker.width(str(img.shape[1])), maker.depth(str(img.shape[2])), ), maker.segmented("0"), # if for segmentation ) # two list for visualization bboxes = [] labels = [] for shape in data['shapes']: class_name = shape['label'] # object name in json file class_id = class_names.index( cn2ens[class_name]) # convert to class id # box info from annotated points xmin = shape['points'][0][0] ymin = shape['points'][0][1] xmax = shape['points'][2][0] ymax = shape['points'][2][1] bboxes.append([xmin, ymin, xmax, ymax]) labels.append(class_id) # xml.append( maker.object( # object info maker.name( cn2ens[shape['label']]), # label name(in english) maker.pose(""), # pose info, doesn't matter maker.truncated("0"), # truncated info, doesn't matter maker.difficult("0"), # diificulty, doesn't matter maker. bndbox( # bbox(up-left corner and bottom-right corner points) maker.xmin(str(xmin)), maker.ymin(str(ymin)), maker.xmax(str(xmax)), maker.ymax(str(ymax)), ), )) # caption for visualize drawing captions = [class_names[l] for l in labels] viz = labelme.utils.draw_instances(img, bboxes, labels, captions=captions) # save it PIL.Image.fromarray(viz).save(out_viz_file) # another visualize format (colored mask in bbox) # convert label to id label_name_to_value = {'背景': 0} for shape in sorted(data['shapes'], key=lambda x: x['label']): label_name = shape['label'] if label_name in label_name_to_value: label_value = label_name_to_value[label_name] else: label_value = len(label_name_to_value) label_name_to_value[label_name] = label_value # labelme's function lbl = utils.shapes_to_label(img.shape, data['shapes'], label_name_to_value) label_names = [None] * (max(label_name_to_value.values()) + 1) for name, value in label_name_to_value.items(): label_names[value] = cn2ens[name] # annotated in english # labelme's function lbl_viz = utils.draw_label(lbl, img, label_names) # save img PIL.Image.fromarray(lbl_viz).save(out_colorize_file) # save voc annotation to xml file with open(out_xml_file, 'wb') as f: f.write(lxml.etree.tostring(xml, pretty_print=True))
def optimize(self, Xs, Ys, Ys_type=None, split_rate=0.7): Ys = reformat_np_arr(Ys, self.estimator.model_Ys_type, from_np_arr_type=Ys_type) dataset = Dataset() dataset.add_data('Xs', Xs) dataset.add_data('Ys', Ys) train_set, test_set = dataset.split((split_rate, 1 - split_rate), shuffle=False) train_Xs, train_Ys = train_set.full_batch() test_Xs, test_Ys = test_set.full_batch() train_Ys = reformat_np_arr(train_Ys, self.estimator.model_Ys_type, from_np_arr_type=Ys_type) test_Ys = reformat_np_arr(test_Ys, self.estimator.model_Ys_type, from_np_arr_type=Ys_type) param_grid_size = self.param_grid_size self.log.info(("optimize [%s], total %s's candidate estimator" % (self.estimator, param_grid_size))) self.result = [] class_ = self.estimator.__class__ gen_param = self.gen_param() for _ in progressbar.progressbar(range(param_grid_size), redirect_stdout=False): try: param = next(gen_param) estimator = class_(**param) estimator.fit(train_Xs, train_Ys) train_score = estimator.score(train_Xs, train_Ys) test_score = estimator.score(test_Xs, test_Ys) predict = estimator.predict(test_Xs) auc_score = sklearn.metrics.roc_auc_score(test_Ys, predict) result = { "train_score": train_score, "test_score": test_score, "param": param, "auc_score": auc_score, } self.result += [result] except KeyboardInterrupt as e: log_error_trace(self.log.error, e) raise KeyboardInterrupt except BaseException as e: self.log.warn(f'while optimize param {param} raise {e}') self.result = sorted(self.result, key=lambda a: ( -a["auc_score"], -a["test_score"], -a["train_score"], )) self.best_param = self.result[0]["param"] estimator = class_(**self.best_param) estimator.fit( train_Xs, train_Ys, ) return estimator
def train(net, criterion, optimizer, dataloaders, epoch, device=torch.device('cuda:0'), hist_args={}): history = History(**hist_args) for e in range(epoch): epoch_loss = 0. for phase in ['train', 'valid']: y_encoder = dataloaders[phase].dataset.y_encoder if phase == 'train': net.train() prefix = '%d, Training: ' % e else: net.eval() prefix = '%d, Validation: ' % e # validation时需要记录所有的真实标签和预测 epoch_label, epoch_loc = [], [] epoch_res_c, epoch_res_s, epoch_res_l = [], [], [] for batch in pb.progressbar(dataloaders[phase], prefix=prefix): imgs, targets = batch[0].to(device), batch[-1].to(device) with torch.set_grad_enabled(phase == 'train'): preds = net(imgs) loss = criterion(preds, targets) if phase == 'train': optimizer.zero_grad() loss.backward() optimizer.step() else: # 如果是eval,需要记录所有的预测,用于计算mAP img_size = list(imgs.shape[-1:-3:-1]) epoch_label.extend(batch[1]) epoch_loc.extend(batch[2]) for pred in preds: res_c, res_s, res_l = y_encoder.decode( pred, img_size) epoch_res_c.append(res_c) epoch_res_s.append(res_s) epoch_res_l.append(res_l) with torch.no_grad(): epoch_loss += loss.item() # 整个epoch的所有batches循环结束 with torch.no_grad(): num_batches = len(dataloaders[phase].dataset) epoch_loss /= num_batches if phase == 'train': history.update_hist(loss=epoch_loss) print('%d, %s, loss: %.4f' % (e, phase, epoch_loss)) else: aps, map_score = mAP(epoch_label, epoch_loc, epoch_res_c, epoch_res_s, epoch_res_l) history.update_hist(val_loss=epoch_loss, mAP=map_score) if map_score is np.nan: map_score = 0 history.update_best(mAP=map_score, loss=epoch_loss, epoch=e, model_wts=copy.deepcopy( net.state_dict())) print('%d, %s, loss: %.4f, mAP: %.4f' % (e, phase, epoch_loss, map_score)) if 'valid' in dataloaders.keys(): net.load_state_dict(history.best[0]['model_wts']) print('valid best loss: %.4f, best mAP: %.4f' % (history.best[0]['loss'], history.best[0]['mAP'])) return history
def __init__(self, collection, resolution=None, margin=0.05, verbose=True): """Constructor of :class:`SaptialIndex` class TODO: update documentation Parameters ---------- features : bbox() + iterable TrackCollection : on construit une grille dont l’emprise est calculée sur la fonction getBBox de TrackCollection et dans un deuxième temps on appelle addSegment([c1,c2], [i,j]) pour chaque segment [c1,c2] (localisé entre les points GPS i et i+1) de chaque trace j, de la collection. Network : on construit une grille (de taille 100 x 100) par défaut, dont l’emprise est calculée sur celle du réseau, et on appelle addSegment ([c1,c2], [i,j]) pour chaque segment [c1,c2] (localisé entre les points GPS i et i+1) de chaque tronçon j, du réseau. resolution : tuple (xsize, ysize) DESCRIPTION. The default is (100, 100). Returns ------- None. """ # Bbox only or collection if isinstance(collection, Bbox): bb = collection else: bb = collection.bbox() bb = bb.copy() bb.addMargin(margin) (self.xmin, self.xmax, self.ymin, self.ymax) = bb.asTuple() ax, ay = bb.getDimensions() if resolution is None: am = max(ax, ay) r = am / 100 resolution = (int(ax / r), int(ay / r)) else: r = resolution resolution = (int(ax / r[0]), int(ay / r[1])) self.collection = collection # Keeps track of registered features self.inventaire = set() # Nombre de dalles par cote self.csize = resolution[0] self.lsize = resolution[1] # print ('nb cellule', self.xsize * self.ysize) # Tableau de collections de features appartenant a chaque dalle. # Un feature peut appartenir a plusieurs dalles. self.grid = [] for i in range(self.csize): self.grid.append([]) for j in range(self.lsize): self.grid[i].append([]) self.dX = ax / self.csize self.dY = ay / self.lsize # Calcul de la grille if isinstance(collection, tuple): self.collection = TrackCollection() return boucle = range(collection.size()) if verbose: print( "Building [" + str(self.csize) + " x " + str(self.lsize) + "] spatial index..." ) boucle = progressbar.progressbar(boucle) for num in boucle: feature = collection[num] # On récupere la trace if isinstance(feature, Track): self.addFeature(feature, num) # On récupère l'arc du reseau qui est une trace elif isinstance(feature, Edge): self.addFeature(feature.geom, num)
def start(self): #Open the image and blur it a little beg = cv2.imread(self.look_path) #print(self.look_path) beg = cv2.GaussianBlur(beg, (self.blur_amount, self.blur_amount), 0) avg_color = average_color(beg) warnings.simplefilter(action='ignore', category=FutureWarning) if self.starting_img == "": blank = np.zeros((beg.shape[0], beg.shape[1], beg.shape[2]), np.uint8) cv2.rectangle(blank, (0, 0), (beg.shape[1], beg.shape[0]), avg_color, -1) else: blank = self.starting_img #cv2.imwrite(my_path + "poo.png", blank) main_dots = [] for i in progressbar(range(self.number_of_dots)): #progress(i, max=self.number_of_dots) difference = cv2.absdiff(beg, blank) if self.diff_blur_amount > 0: difference = cv2.GaussianBlur( difference, (self.diff_blur_amount, self.diff_blur_amount), 0) (minVal, maxVal, minLoc, maxLoc) = cv2.minMaxLoc(gray_img(difference)) pos = (beg[maxLoc[1], maxLoc[0]]) b, g, r = pos dot = dot_series.Dot((maxLoc[1], maxLoc[0]), 1, (b, g, r)) changer = 0 inactive_count = 0 while inactive_count < 10: if changer == 0: choises = [ copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot) ] choises[1].radius += 1 choises[1].radius += 5 choises[2].radius += 10 choises[3].radius = max(choises[2].radius - 1, 1) choises[4].radius = max(choises[2].radius - 5, 1) dot, winner = find_winner(choises, dot, main_dots, beg, blank) if winner == 0: inactive_count += 1 else: inactive_count = 0 elif changer == 1: choises = [ copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot) ] choises[1].color = (min(choises[1].color[0] + 5, 255), choises[1].color[1], choises[1].color[2]) choises[1].color = (min(choises[1].color[0] + 1, 255), choises[1].color[1], choises[1].color[2]) choises[1].color = (max(choises[1].color[0] - 5, 0), choises[1].color[1], choises[1].color[2]) choises[1].color = (max(choises[1].color[0] - 1, 0), choises[1].color[1], choises[1].color[2]) dot, winner = find_winner(choises, dot, main_dots, beg, blank) if winner == 0: inactive_count += 1 else: inactive_count = 0 elif changer == 2: choises = [ copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot) ] choises[1].color = (choises[1].color[0], min(choises[1].color[1] + 5, 255), choises[1].color[2]) choises[1].color = (choises[1].color[0], min(choises[1].color[1] + 1, 255), choises[1].color[2]) choises[1].color = (choises[1].color[0], max(choises[1].color[1] - 5, 0), choises[1].color[2]) choises[1].color = (choises[1].color[0], max(choises[1].color[1] - 1, 0), choises[1].color[2]) dot, winner = find_winner(choises, dot, main_dots, beg, blank) if winner == 0: inactive_count += 1 else: inactive_count = 0 elif changer == 3: choises = [ copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot) ] choises[1].color = (choises[1].color[0], choises[1].color[1], min(choises[1].color[2] + 5, 255)) choises[1].color = (choises[1].color[0], choises[1].color[1], min(choises[1].color[2] + 1, 255)) choises[1].color = (choises[1].color[0], choises[1].color[1], max(choises[1].color[2] - 5, 0)) choises[1].color = (choises[1].color[0], choises[1].color[1], max(choises[1].color[2] - 1, 0)) dot, winner = find_winner(choises, dot, main_dots, beg, blank) if winner == 0: inactive_count += 1 else: inactive_count = 0 elif changer == 4: choises = [ copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot) ] choises[1].position = (choises[1].position[0] + 1, choises[1].position[1]) choises[2].position = (choises[2].position[0] - 1, choises[2].position[1]) choises[3].position = (choises[3].position[0], choises[3].position[1] + 1) choises[4].position = (choises[4].position[0], choises[4].position[1] - 1) choises[5].position = (choises[1].position[0] + 5, choises[1].position[1]) choises[6].position = (choises[2].position[0] - 5, choises[2].position[1]) choises[7].position = (choises[3].position[0], choises[3].position[1] + 5) choises[8].position = (choises[4].position[0], choises[4].position[1] - 5) choises[9].position = (choises[1].position[0] + 1, choises[1].position[1] + 1) choises[10].position = (choises[2].position[0] - 1, choises[2].position[1] + 1) choises[11].position = (choises[3].position[0] + 1, choises[3].position[1] - 1) choises[12].position = (choises[4].position[0] - 1, choises[4].position[1] - 1) choises[13].position = (choises[1].position[0] + 5, choises[1].position[1] + 5) choises[14].position = (choises[2].position[0] - 5, choises[2].position[1] + 5) choises[15].position = (choises[3].position[0] + 5, choises[3].position[1] - 5) choises[16].position = (choises[4].position[0] - 5, choises[4].position[1] - 5) dot, winner = find_winner(choises, dot, main_dots, beg, blank) if winner == 0: inactive_count += 1 else: inactive_count = 0 else: choises = [ copy.deepcopy(dot), copy.deepcopy(dot), copy.deepcopy(dot) ] choises[1].infront = True choises[2].infront = False dot, winner = find_winner(choises, dot, main_dots, beg, blank) if winner == 0: inactive_count += 1 else: inactive_count = 0 changer += 1 if changer == 6: changer = 0 if dot.infront: main_dots.append(dot) else: main_dots.insert(0, dot) blank = plot_dots(blank, main_dots) #difference = cv2.absdiff(beg, blank) cv2.imwrite(self.final_path, blank) """ print(main_dots[0].position) print(main_dots[0].radius) print(main_dots[0].color) cv2.imshow("beg", beg) cv2.imshow("blank", blank) cv2.imshow("diff", difference) cv2.waitKey(0) cv2.destroyAllWindows() """ #plot_dots(img, main_dots) #progress(self.number_of_dots, max=self.number_of_dots) print("") return blank
def _calc_test_hvps_log_loss(criterion: Callable, trained_model: nn.Module, train_dataloader: DeviceDataLoader, test_dataloader: DataLoader, run_params: Mapping, diff_wrt: Optional[torch.Tensor] = None, show_progress: bool = False): device = train_dataloader.device diff_wrt = maybe( diff_wrt, [p for p in trained_model.parameters() if p.requires_grad]) if run_params['use_gauss_newton']: matmul_class: Union[Type[GNP], Type[HVP]] = GNP damping = 0.001 else: matmul_class = HVP damping = 0.01 matmul = matmul_class( calc_loss=lambda xs, target: criterion(trained_model(*xs), target), parameters=diff_wrt, data=train_dataloader, data_len=len(train_dataloader), damping=damping, cache_batch=True) cg = CG(matmul=matmul, result_len=sum(p.numel() for p in diff_wrt), max_iters=maybe(run_params['max_cg_iters'], sum(p.numel() for p in diff_wrt))) test_hvps: List[torch.Tensor] = [] iterator = progressbar( test_dataloader) if show_progress else test_dataloader for batch in iterator: x_test, target = to_device(batch, device) loss_at_x_test = criterion(trained_model(*x_test), target.squeeze()) grads = autograd.grad(loss_at_x_test, diff_wrt) grad_at_z_test = collect(grads) if len(test_hvps) != 0: init = test_hvps[-1].detach().clone() else: init = torch.zeros_like(grad_at_z_test) def _min(x, grad_at_z_test=grad_at_z_test): x_tens = torch.tensor( x, device=device) if not isinstance(x, torch.Tensor) else x grad_tens = torch.tensor( grad_at_z_test, device=device) if not isinstance( grad_at_z_test, torch.Tensor) else grad_at_z_test return np.array(0.5 * matmul(x_tens).dot(x_tens) - grad_tens.dot(x_tens)) def _grad(x, grad_at_z_test=grad_at_z_test): x_tens = torch.tensor( x, device=device) if not isinstance(x, torch.Tensor) else x return np.array(matmul(x_tens) - grad_at_z_test) def _hess(x, p): grad_tens = torch.tensor( p, device=device) if not isinstance(p, torch.Tensor) else p return np.array(matmul(grad_tens)) if getattr(run_params, 'use_scipy', False): test_hvps.append( torch.tensor(fmin_ncg(f=_min, x0=init, fprime=_grad, fhess_p=_hess, avextol=1e-8, maxiter=100, disp=False), device=device)) else: test_hvps.append( cg.solve(grad_at_z_test, test_hvps[-1] if len(test_hvps) != 0 else None)) matmul.clear_batch() return torch.stack(test_hvps)
def run(self, processes=None): all_nodes = list(self.network.A['physical']) for i in progressbar.progressbar(range(self.tmin, self.tmax)): self.run_chain(all_nodes, processes, len(all_nodes)) self.times.append(i)
def get_test_dataset_iterator(display_progress=False): iterator = read_dataset_from_csv(test_dataset_path, columns=COLUMNS, chunk_size=TESTING_CHUNK_SIZE) if display_progress: iterator = progressbar(iterator, max_value=TESTING_CHUNKS_COUNT, max_error=False) return get_dataset_iterator(iterator)
def add_task(task_name, dataset_name=None, local_path=None, dataset_address=None, preproc_address=None, include_predictors=None, exclude_predictors=None, reingest=False, scan_length=1000, dataset_summary=None, url=None, task_summary=None, **kwargs): """ Adds a BIDS dataset task to the database. Args: task_name - task to add dataset_name - overide dataset name local_path - path to local bids dataset. dataset_address - remote address of BIDS dataset. preproc_address - remote address of preprocessed files. include_predictors - set of predictors to ingest exclude_predictors - set of predictors to exclude from ingestions reingest - force reingesting even if dataset already exists scan_length - default scan length in case it cant be found in image dataset_summary - Dataset summary description, url - Dataset external link, task_summary - Task summary description, kwargs - arguments to filter runs by Output: dataset model id """ cache.clear() if dataset_address is not None and local_path is None: local_path = install( source=dataset_address, path=(Path( current_app.config['DATASET_DIR']) / dataset_name).as_posix() ).path local_path = Path(local_path) assert isfile(str(local_path / 'dataset_description.json')) layout = BIDSLayout(str(local_path), derivatives=True) if task_name not in layout.get_tasks(): raise ValueError("Task {} not found in dataset {}".format( task_name, local_path)) dataset_name = dataset_name if dataset_name is not None \ else layout.description['Name'] # Get or create dataset model from mandatory arguments dataset_model, new_ds = get_or_create(Dataset, name=dataset_name) if new_ds: dataset_model.description = layout.description dataset_model.summary = dataset_summary dataset_model.url = url dataset_model.dataset_address = dataset_address dataset_model.preproc_address = preproc_address dataset_model.local_path = local_path.as_posix() db.session.commit() elif not reingest: print("Dataset found, skipping ingestion...") # return dataset_model.id # Get or create task task_model, new_task = get_or_create( Task, name=task_name, dataset_id=dataset_model.id) if new_task: task_model.description = json.load( (local_path / 'task-{}_bold.json'.format(task_name)).open()) task_model.summary = task_summary task_model.TR = task_model.description['RepetitionTime'] db.session.commit() stims_processed = {} """ Parse every Run """ print("Parsing runs") ''' all_runs = layout.get(task=task_name, suffix='bold', extensions='.nii.gz', desc=None, **kwargs) ''' # this worked one above didn't all_runs = layout.get(task=task_name, suffix='bold', extensions='.nii.gz') for img in progressbar(all_runs): """ Extract Run information """ # Get entities entities = {entity: getattr(img, entity) for entity in ['subject', 'session', 'acquisition'] if entity in img.entities} run_number = img.run if hasattr(img, 'run') else None run_model, new = get_or_create( Run, dataset_id=dataset_model.id, number=run_number, task_id=task_model.id, **entities) entities['task'] = task_model.name if run_number: run_number = str(run_number).zfill(2) entities['run'] = run_number # Get duration (helps w/ transformations) if img.image is not None: run_model.duration = img.image.shape[3] * \ img.image.header.get_zooms()[-1] else: run_model.duration = scan_length # Put back as int if 'run' in entities: entities['run'] = int(entities['run']) """ Extract Predictors""" # Assert event files exist (for DataLad) for e in layout.get_nearest( img.path, suffix='events', all_=True, strict=False): assert isfile(e) collection = layout.get_collections( 'run', scan_length=run_model.duration, **entities)[0] if 'stim_file' in collection.variables: stims = collection.variables.pop('stim_file') else: stims = None add_predictor_collection( collection, dataset_model.id, run_model.id, include=include_predictors, exclude=exclude_predictors, TR=task_model.TR) """ Ingest Stimuli """ if stims is not None: for i, val in enumerate(stims.values): stim_path = local_path / 'stimuli' / val if val not in stims_processed: try: stim_hash = hash_stim(stim_path) except OSError: current_app.logger.debug( '{} not found.'.format(stim_path)) continue stims_processed[val] = stim_hash else: stim_hash = stims_processed[val] stim_model, _ = add_stimulus( stim_hash, path=stim_path, dataset_id=dataset_model.id) # Get or create Run Stimulus association runstim, _ = get_or_create( RunStimulus, stimulus_id=stim_model.id, run_id=run_model.id, onset=stims.onset.tolist()[i], duration=stims.duration.tolist()[i]) """ Add GroupPredictors """ print("Adding group predictors") add_group_predictors(dataset_model.id, local_path / 'participants.tsv') return dataset_model.id
print('Generating main pvd file') with open(main_filename,'w') as f: f.write('<?xml version="1.0"?>\n') f.write('<VTKFile type="Collection" version="0.1"\n') f.write(' byte_order="LittleEndian"\n') f.write(' compressor="vtkZLibDataCompressor">\n') f.write(' <Collection>\n') filename = "pts_{:010d}.vtp".format(0) if isfile(filename): f.write(f'<DataSet timestep="0" group="" part="0"\n') f.write(f' file="{filename}"/>\n') for filenumber in progressbar.progressbar(filenumbers): filename = "pts_{:010d}.vtp".format(filenumber) if isfile(filename): f.write(f'<DataSet timestep="{np.round(filenumber*timestep,10)}" group="" part="0"\n') f.write(f' file="{filename}"/>\n') f.write(' </Collection>\n') f.write('</VTKFile>\n')
#%% import random import string import pandas as pd import progressbar from fuzzywuzzy import process from multiprocessing.dummy import Pool as ThreadPool max_thread = 4 #%% df = pd.DataFrame(index=range(15000), columns=['to_match']) df_opt = pd.DataFrame(index=range(28000), columns=['option']) for i, row in progressbar.progressbar(df.iterrows(), max_value=len(df)): length = random.randint(5, 10) df.at[i, 'to_match'] = ''.join( random.choices(string.ascii_uppercase, k=length)) for i, row in progressbar.progressbar(df_opt.iterrows(), max_value=len(df_opt)): length = random.randint(5, 10) df_opt.at[i, 'option'] = ''.join( random.choices(string.ascii_uppercase, k=length)) df_opt = df_opt['option'] #%% df['score'] = 'None' df['match'] = 'None' l_tpl_lev = []
def wait(): for i in progressbar.progressbar(range(1800)): sleep(1)
import time import progressbar for i in progressbar.progressbar(range(100)): time.sleep(0.02)
def __init__(self, **kwargs): import progressbar as pb self.progressbarvar = pb.progressbar(**kwargs) return None
def reset_ids(self, target_classes, import_objects, import_objects_by_type_and_id, existing_classes, map_treenodes=True, save=True): """Reset the ID of each import object to None so that a new object will be created when the object is saved. At the same time an index is created that allows per-type lookups of foreign key fields """ logger.info("Building foreign key update index") # Build index for foreign key fields in models. For each type, map # each foreign key name to a model class. fk_index = defaultdict(dict) for c in target_classes: class_index = fk_index[c] foreign_key_fields = [ f for f in c._meta.get_fields() if f.is_relation and f.many_to_one # ForeignKey instances #if field.get_internal_type() == 'ForeignKey': and f.related_model in target_classes ] for field in foreign_key_fields: # Get the database column name for this field class_index[field.attname] = field.related_model logger.info("Updating foreign keys to imported objects with new IDs") all_classes = dict() all_classes.update(existing_classes) updated_fk_ids = 0 unchanged_fk_ids = 0 explicitly_created_summaries = 0 other_tasks = set(import_objects.keys()) - set(ordered_save_tasks) # Iterate objects to import and respect dependency order for object_type in ordered_save_tasks + list(other_tasks): objects = import_objects.get(object_type) if not objects: # No objects of this object type are imported continue fk_fields = fk_index[object_type] # No need to do rest if there are no foreign keys to change to begin # with. if len(fk_fields) == 0: continue imported_parent_nodes = [] bar_prefix = "- {}: ".format(object_type.__name__) for deserialized_object in progressbar.progressbar(objects, max_value=len(objects), redirect_stdout=True, prefix=bar_prefix): obj = deserialized_object.object obj_type = type(obj) for fk_field, fk_type in six.iteritems(fk_fields): # Get import object with the former ID referenced in # this field. current_ref = getattr(obj, fk_field) # Get updated model objects of the referenced type imported_objects_by_id = import_objects_by_type_and_id[fk_type] ref_obj = imported_objects_by_id.get(current_ref) if ref_obj: # Update foreign key reference to ID of newly saved # object. Only for treenodes this is expected to result # in not yet available data if object_type == Treenode and fk_type == Treenode: imported_parent_nodes.append((obj, current_ref)) elif ref_obj.id is None: raise ValueError("The referenced {} object '{}' with import ID {} wasn't stored yet".format( fk_type, str(ref_obj), current_ref)) setattr(obj, fk_field, ref_obj.id) updated_fk_ids += 1 else: unchanged_fk_ids += 1 # Save objects if they should either be imported or have change # foreign key fields if save and (updated_fk_ids or obj.id is None): obj.save() # Treenodes are special, because they can reference themselves. They # need therefore a second iteration of reference updates after all # treenodes have been saved and new IDs are available. if map_treenodes and object_type == Treenode: logger.info('Mapping parent IDs of treenodes to imported data') imported_objects_by_id = import_objects_by_type_and_id[Treenode] for obj, parent_id in progressbar.progressbar(imported_parent_nodes, max_value=len(imported_parent_nodes), redirect_stdout=True, prefix="- Mapping parent treenodes: "): new_parent = imported_objects_by_id.get(parent_id) if not new_parent: raise ValueError("Could not find imported treenode {}".format(parent_id)) obj.parent_id = new_parent.id if save: obj.save() # Update list of known classes after new classes have been saved if object_type == Class: for deserialized_object in objects: obj = deserialized_object.object all_classes[obj.class_name] = obj.id # If skeleton class instances are created, make sure the skeleton # summary table entries for the respective skeletons are there. # Otherwise the ON CONFLICT claues of the summary update updates can # be called multiple times. The alternative is to disable the # trigger during import. pre_create_summaries = False if object_type == ClassInstance and pre_create_summaries: skeleton_class_id = all_classes.get('skeleton') for deserialized_object in objects: obj = deserialized_object.object if obj.class_column_id == skeleton_class_id: r = SkeletonSummary.objects.get_or_create(project=self.target, skeleton_id=obj.id) explicitly_created_summaries += 1 logger.info("".join(["{} foreign key references updated, {} did not ", "require change, {} skeleton summaries were created"]).format( updated_fk_ids, unchanged_fk_ids, explicitly_created_summaries))
coords = pandas.read_csv(coords_file, header=None, delim_whitespace=True) print('Reading boundary codes') codes = pandas.read_csv(boundary_label_file, header=None, delim_whitespace=True) # In[9]: print('Passing coordinates to vtk') pts = vtk.vtkPoints() pts.SetNumberOfPoints(coords.shape[0]) vv = coords.values for i in progressbar.progressbar(range(coords.shape[0])): pts.SetPoint(i, vv[i,1:]) # In[25]: print('Passing boundaries to vtk') boundaries = vtk.vtkCellArray() with open(boundary_file,'r') as f: for line in f: data = line.strip().split()
import time import progressbar import numpy as np for i in progressbar.progressbar(np.arange(100)): time.sleep(0.02) with progressbar.ProgressBar(max_value=10) as bar: for i in range(10): time.sleep(0.1) bar.update(i)
# print('success_list: {}'.format(success_list)) # print('norm_success: {}'.format(norm_success)) ax.plot(alpha, norm_success, label='N: {}'.format(N)) if __name__ == "__main__": # Run your code in order to study Perceptron training at least for the following parameter settings: # N = 20, P = αN with α = 0.75, 1.0, 1.25, . . . 3.0, nD =50, n_max =100 n_max = 100 nD = 50 N_array = [5, 10, 15, 20, 50, 100] # np.random.seed(0) # To make reproducible sets (if needed) # determine the value of the fraction of successful runs as a function of alpha=P/N alpha = np.arange(0.75, 3.25, 0.25) fig = plt.figure() ax = plt.subplot(111) for N in progressbar.progressbar(N_array): test_runs(n_max, nD, N, alpha, ax) plt.xlabel(r'$\alpha = P/N$') plt.ylabel(r'$Q_{l.s.}$') ax.legend() fig.savefig('Lab1/graphs/Q-alpha-graph.png') plt.show()
line = f.readline().split() nelement_types = int(line[0]) nelements = 0 for k in range(nelement_types): line = f.readline().split() nelements = nelements + int(line[1]) f.readline() #1.0 1.0 f.readline() #fensap input file pts = vtk.vtkPoints() pts.SetNumberOfPoints(npoints) for i in progressbar(range(npoints)): line = f.readline().split() pts.SetPoint(i, float(line[0]), float(line[1]), float(line[2])) cells = vtk.vtkCellArray() celltypes = [] c = 0 with ProgressBar(max_value=nelements) as bar: while True: line = f.readline() if line=='': break line = line.split() nverts = len(line)
#! /usr/bin/env python3 import os, sys, timeit, progressbar # Get command-line arguments (F, N), T = map(int, sys.argv[1:3]), float(sys.argv[3]) # Generate a random file hierarchy print(f'1. Generate {F} random files for testing:') for f in progressbar.progressbar(range(0, F)): os.system(f'mkdir -p test; dd if=/dev/urandom of=test/{f} bs=1 count=1024 > /dev/null 2>&1') # Mesure 10 execution of a call to md5sumsum on our test dataset print(f'2. Mesure {N} executions of our script:') x = timeit.timeit(""" import os os.system('./md5sumsum test') """, number=N) # Return an error exit code to Travis CI if our script is too slow print(f'Our execution time is {x}s, our test goal was {T}s') sys.exit(1 if x > T else 0)
wr.SetFileName(filename) wr.Write() bar = progressbar.ProgressBar(max_value=len(unique_times)) with mp.Pool(processes = ncpus) as p: for i, _ in enumerate(p.imap_unordered(save_one_file, range(len(unique_times))), 1): bar.update(i) print('Generating main pvd file') with open(main_filename,'w') as f: f.write('<?xml version="1.0"?>\n') f.write('<VTKFile type="Collection" version="0.1"\n') f.write(' byte_order="LittleEndian"\n') f.write(' compressor="vtkZLibDataCompressor">\n') f.write(' <Collection>\n') for i in progressbar.progressbar(range(len(unique_times))): if empty_file[i]==0: filename = "pts_{:010d}.vtp".format(i) f.write(f'<DataSet timestep="{unique_times[i]}" group="" part="0"\n') f.write(f' file="{filename}"/>\n') f.write(' </Collection>\n') f.write('</VTKFile>\n')
def perceptron_forest_study(data, trees, fracs, num_seeds=5, modes=["with", "without"], filename=None): """ Generate an accuracy vs number of decision tree plot for a perceptron forest. Parameters ---------- data : (pandas.DataFrame, pandas.DataFrame) Input data for the learning algorithms. Tuple ordered by data and classifications. trees : numpy.ndarray Array containing number of trees to try in perceptron forest. fracs : numpy.ndarray Array containing fraction of training data each tree should be trained on. num_seeds : int, optional Number of random seeds to average results over. The default is 5. modes: list(str), len 1-2 List of modes to run. Options are 'with' for running with data replacement and 'without' for running without data replacment. The default is both (['with', 'without']). filename : str, optional File to save plot to. The default is None. Returns ------- t_w : int Number of trees that optimizes accuracy with replacement. f_w : float Data fraction that optimizes accuracy with replacement. t_w0 : int Number of trees that optimizes accuracy without replacement. f_w0 : float Data fraction that optimizes accuracy without replacement. """ # Define random seeds and parameters np.random.seed( 0) # Set seed of random number generator for reproducability seeds = np.random.randint(0, 1E9, num_seeds) # Loop through number of trees in PF means_w, means_wo = [], [] errs_w, errs_wo = [], [] print("\nPerceptron Forest Study\n") for n in progressbar(trees): for frac in fracs: # Define hyperparams # w means with replacement, wo means without replacement hyperparams_w = { "n_estimators": n, "max_samples": frac, "bootstrap": True } hyperparams_wo = { "n_estimators": n, "max_samples": frac, "bootstrap": False } # Loop through seeds mean_w, var_w = 0.0, 0.0 mean_wo, var_wo = 0.0, 0.0 for seed in seeds: if "with" in modes: # Init learner hyperparams_w["random_state"] = seed PF_w = Project(data, "perceptron_forest", hyperparams_w) # Evaluate model accuracy d_w = PF_w.evaluate_model_accuracy() # Tally mean and std to averaged quantities over seed mean_w += d_w[0][1] var_w += d_w[0][2]**2 if "without" in modes: hyperparams_wo["random_state"] = seed PF_wo = Project(data, "perceptron_forest", hyperparams_wo) d_wo = PF_wo.evaluate_model_accuracy() mean_wo += d_wo[0][1] var_wo += d_wo[0][2]**2 # Add to list if "with" in modes: means_w.append(mean_w / num_seeds) errs_w.append(d_w[0][3] * np.sqrt(var_w) / num_seeds) if "without" in modes: means_wo.append(mean_wo / num_seeds) errs_wo.append(d_wo[0][3] * np.sqrt(var_wo) / num_seeds) if "with" in modes: # Convert everything to np arrays means_w = np.array(means_w) means_wo = np.array(means_wo) # Make sure error bars are in (0,1) acc_w_upper = means_w + errs_w acc_w_lower = means_w - errs_w acc_w_upper[acc_w_upper > 1.0] = 1.0 acc_w_lower[acc_w_lower < 0.0] = 0.0 if "without" in modes: errs_w = np.array(errs_w) errs_wo = np.array(errs_wo) acc_wo_upper = means_wo + errs_wo acc_wo_lower = means_wo - errs_wo acc_wo_upper[acc_wo_upper > 1.0] = 1.0 acc_wo_lower[acc_wo_lower < 0.0] = 0.0 # Make mesh of combination of fractions and trees f, t = np.meshgrid(fracs, trees) f = f.flatten() t = t.flatten() fig = plt.figure() ax = fig.add_subplot(111, projection='3d') ax.set_xlabel("Number of Decision Trees") ax.set_ylabel("Data Fraction / Tree") ax.set_zlabel("Average Mean Accuracy") if "with" in modes: ax.scatter(t, f, means_w, marker="o", s=2, color="blue", label="With Replacement") # Add error bars for i in range(t.size): x = t[i] y = f[i] low_w = acc_w_lower[i] high_w = acc_w_upper[i] ax.plot([x, x], [y, y], [low_w, high_w], marker="_", color="blue") if "without" in modes: ax.scatter(t, f, means_wo, marker="o", s=2, color="green", label="Without Replacement") # Add error bars for i in range(t.size): x = t[i] y = f[i] low_wo = acc_wo_lower[i] high_wo = acc_wo_upper[i] ax.plot([x, x], [y, y], [low_wo, high_wo], marker="_", color="green") ax.legend(loc="best", markerscale=2) ax.xaxis.set_major_locator(MaxNLocator(integer=True)) if filename is not None: # Make sure directory exists file_dir = filename[:filename.rfind("/")] if "/" in filename and not os.path.isdir(file_dir): os.makedirs(file_dir) fig.savefig(filename, bbox_inches="tight") filebase, extention = filename.rsplit(".", 1) pl.dump(fig, open(filebase + ".fig", "wb")) else: plt.show() best_params = [] # Determine optimal hyper-parameters for best accuracy if "with" in modes: best_acc_w = acc_w_lower.max() best_match_w = acc_w_upper[acc_w_upper - best_acc_w > 0].min() t_w, f_w = t[acc_w_upper == best_match_w].min(), f[acc_w_upper == best_match_w].min() best_params.append(t_w) best_params.append(f_w) if "without" in modes: best_acc_wo = acc_wo_lower.max() best_match_wo = acc_wo_upper[acc_wo_upper - best_acc_wo > 0].min() t_wo, f_wo = t[acc_wo_upper == best_match_wo].min(), f[ acc_wo_upper == best_match_wo].min() best_params.append(t_wo) best_params.append(f_wo) return best_params
f.write(b'NONE000\0') #2 f.write(b'NONE000\0') #3 f.write(b'NONE000\0') #4 f.write(b'NONE000\0') #5 f.write(b'NONE000\0') #6 f.write(b'NONE000\0') #7 f.write(b'NONE000\0') #8 f.write(b'NONE000\0') #9 f.write(b'NONE000\0') #10 vector.astype('float64').tofile(f) corrected_times = np.round(t20-t20[0], 15) for i in progressbar.progressbar(range(t20.shape[0])): t = t20[i] rdr.SetTimeValue(t) rdr.Update() mesh = rdr.GetOutput().GetBlock(0) veloc = mesh.GetPointData().GetArray(varname) v = np.zeros((mesh.GetNumberOfPoints(),ndimensions), dtype = np.float64) for j in range(veloc.GetNumberOfTuples()): vv = veloc.GetTuple3(j) v[j,:] = vv[:ndimensions] filename = '{:s}-XFIEL.{:08d}.{:08d}.mpio.bin'.format(problem_name, field_id, i+1) write_vector_mpio( os.path.join( outpath, filename), v, corrected_times[i] )
def buildSearchIndices(project_number, section, mode='index', chunk=None, max_age_days=None): """ Builds search index for a project. section must be one of: 'index' - create the index for the project if it does not exist 'mappings' - create mappings for the project if they do not exist 'media' - create documents for media 'states' - create documents for states 'localizations' - create documents for localizations 'treeleaves' - create documents for treeleaves """ project_name = Project.objects.get(pk=project_number).name logger.info(f"Building search indices for project {project_number}: {project_name}") if section == 'index': # Create indices logger.info("Building index...") TatorSearch().create_index(project_number) logger.info("Build index complete!") return if section == 'mappings': # Create mappings logger.info("Building mappings for media types...") for type_ in progressbar(list(MediaType.objects.filter(project=project_number))): TatorSearch().create_mapping(type_) logger.info("Building mappings for localization types...") for type_ in progressbar(list(LocalizationType.objects.filter(project=project_number))): TatorSearch().create_mapping(type_) logger.info("Building mappings for state types...") for type_ in progressbar(list(StateType.objects.filter(project=project_number))): TatorSearch().create_mapping(type_) logger.info("Building mappings for leaf types...") for type_ in progressbar(list(LeafType.objects.filter(project=project_number))): TatorSearch().create_mapping(type_) logger.info("Build mappings complete!") return class DeferredCall: def __init__(self, qs): self._qs = qs def __call__(self): for entity in self._qs.iterator(): for doc in TatorSearch().build_document(entity, mode): yield doc # Get queryset based on selected section. logger.info(f"Building documents for {section}...") qs = CLASS_MAPPING[section].objects.filter(project=project_number, meta__isnull=False) # Apply max age filter. if max_age_days: min_modified = datetime.datetime.now() - datetime.timedelta(days=max_age_days) qs = qs.filter(modified_datetime__gte=min_modified) # Apply limit/offset if chunk parameter given. if chunk is not None: offset = INDEX_CHUNK_SIZE * chunk qs = qs.order_by('id')[offset:offset+INDEX_CHUNK_SIZE] batch_size = 500 count = 0 bar = ProgressBar(redirect_stderr=True, redirect_stdout=True) dc = DeferredCall(qs) total = qs.count() bar.start(max_value=total) for ok, result in streaming_bulk(TatorSearch().es, dc(),chunk_size=batch_size, raise_on_error=False): action, result = result.popitem() if not ok: print(f"Failed to {action} document! {result}") bar.update(min(count, total)) count += 1 if count > total: print(f"Count exceeds list size by {total - count}") bar.finish()
def _get_detections(generator, model, score_threshold=0.05, max_detections=100, save_path=None): """ Get the detections from the model using the generator. The result is a list of lists such that the size is: all_detections[num_images][num_classes] = detections[num_class_detections, 5] Args: generator: The generator used to run images through the model. model: The model to run on the images. score_threshold: The score confidence threshold to use. max_detections: The maximum number of detections to use per image. save_path: The path to save the images with visualized detections to. Returns: A list of lists containing the detections for each image in the generator. """ all_detections = [[ None for i in range(generator.num_classes()) if generator.has_label(i) ] for j in range(generator.size())] for i in progressbar.progressbar(range(generator.size()), prefix='Running network: '): raw_image = generator.load_image(i) image = generator.preprocess_image(raw_image.copy()) image, scale = generator.resize_image(image) # run network boxes, scores, labels = model.predict_on_batch( np.expand_dims(image, axis=0))[:3] # correct boxes for image scale boxes /= scale # select indices which have a score above the threshold indices = np.where(scores[0, :] > score_threshold)[0] # select those scores scores = scores[0][indices] # find the order with which to sort the scores scores_sort = np.argsort(-scores)[:max_detections] # select detections # (n, 4) image_boxes = boxes[0, indices[scores_sort], :] # (n, ) image_scores = scores[scores_sort] # (n, ) image_labels = labels[0, indices[scores_sort]] # (n, 6) image_detections = np.concatenate([ image_boxes, np.expand_dims(image_scores, axis=1), np.expand_dims(image_labels, axis=1) ], axis=1) if save_path is not None: draw_annotations(raw_image, generator.load_annotations(i), label_to_name=generator.label_to_name) draw_detections(raw_image, image_boxes, image_scores, image_labels, label_to_name=generator.label_to_name, score_threshold=score_threshold) cv2.imwrite(os.path.join(save_path, '{}.png'.format(i)), raw_image) # copy detections to all_detections for label in range(generator.num_classes()): if not generator.has_label(label): continue all_detections[i][label] = image_detections[ image_detections[:, -1] == label, :-1] return all_detections
def _get_detections(generator, model, score_threshold=0.05, max_detections=100, visualize=False): """ Get the detections from the model using the generator. The result is a list of lists such that the size is: all_detections[num_images][num_classes] = detections[num_class_detections, 5] Args: generator: The generator used to run images through the model. model: The model to run on the images. score_threshold: The score confidence threshold to use. max_detections: The maximum number of detections to use per image. save_path: The path to save the images with visualized detections to. Returns: A list of lists containing the detections for each image in the generator. """ all_detections = [[ None for i in range(generator.num_classes()) if generator.has_label(i) ] for j in range(generator.size())] for i in progressbar.progressbar(range(generator.size()), prefix='Running network: '): image = generator.load_image(i) src_image = image.copy() h, w = image.shape[:2] anchors = generator.anchors image, scale, offset_h, offset_w = generator.preprocess_image(image) # run network boxes, scores, labels = model.predict_on_batch( [np.expand_dims(image, axis=0), np.expand_dims(anchors, axis=0)]) boxes[..., [0, 2]] = boxes[..., [0, 2]] - offset_w boxes[..., [1, 3]] = boxes[..., [1, 3]] - offset_h boxes /= scale boxes[:, :, 0] = np.clip(boxes[:, :, 0], 0, w - 1) boxes[:, :, 1] = np.clip(boxes[:, :, 1], 0, h - 1) boxes[:, :, 2] = np.clip(boxes[:, :, 2], 0, w - 1) boxes[:, :, 3] = np.clip(boxes[:, :, 3], 0, h - 1) # select indices which have a score above the threshold indices = np.where(scores[0, :] > score_threshold)[0] # select those scores scores = scores[0][indices] # find the order with which to sort the scores scores_sort = np.argsort(-scores)[:max_detections] # select detections # (n, 4) image_boxes = boxes[0, indices[scores_sort], :] # (n, ) image_scores = scores[scores_sort] # (n, ) image_labels = labels[0, indices[scores_sort]] # (n, 6) detections = np.concatenate([ image_boxes, np.expand_dims(image_scores, axis=1), np.expand_dims(image_labels, axis=1) ], axis=1) if visualize: draw_annotations(src_image, generator.load_annotations(i), label_to_name=generator.label_to_name) draw_detections(src_image, detections[:5, :4], detections[:5, 4], detections[:5, 5].astype(np.int32), label_to_name=generator.label_to_name, score_threshold=score_threshold) # cv2.imwrite(os.path.join(save_path, '{}.png'.format(i)), raw_image) cv2.namedWindow('{}'.format(i), cv2.WINDOW_NORMAL) cv2.imshow('{}'.format(i), src_image) cv2.waitKey(0) # copy detections to all_detections for class_id in range(generator.num_classes()): all_detections[i][class_id] = detections[detections[:, -1] == class_id, :-1] return all_detections
def import_data(self): """ Imports data from a file and overrides its properties, if wanted. This method also deactivates auto commit (if it is activated) temporary. """ cursor = connection.cursor() # Defer all constraint checks cursor.execute('SET CONSTRAINTS ALL DEFERRED') # Drop summary table trigger to make insertion faster cursor.execute(""" DROP TRIGGER on_edit_treenode_update_summary_and_edges ON treenode; DROP TRIGGER on_insert_treenode_update_summary_and_edges ON treenode; DROP TRIGGER on_delete_treenode_update_summary_and_edges ON treenode; """) # Get all existing users so that we can map them basedon their username. mapped_user_ids = set() mapped_user_target_ids = set() # Map data types to lists of object of the respective type import_data = defaultdict(list) n_objects = 0 # Read the file and sort by type logger.info("Loading data from {}".format(self.source)) with open(self.source, "r") as data: loaded_data = serializers.deserialize(self.format, data) for deserialized_object in progressbar.progressbar(loaded_data, max_value=progressbar.UnknownLength, redirect_stdout=True): obj = deserialized_object.object import_data[type(obj)].append(deserialized_object) n_objects += 1 if n_objects == 0: raise CommandError("Nothing to import, no importable data found") created_users = dict() if import_data.get(User): import_users = dict((u.object.id, u) for u in import_data.get(User)) logger.info("Found {} referenceable users in import data".format(len(import_users))) else: import_users = dict() logger.info("Found no referenceable users in import data") # Get CATMAID model classes, which are the ones we want to allow # optional modification of user, project and ID fields. app = apps.get_app_config('catmaid') user_updatable_classes = set(app.get_models()) logger.info("Adjusting {} import objects to target database".format(n_objects)) # Needed for name uniquness of classes, class_instances and relations existing_classes = dict(Class.objects.filter(project_id=self.target.id) \ .values_list('class_name', 'id')) existing_relations = dict(Relation.objects.filter(project_id=self.target.id) \ .values_list('relation_name', 'id')) existing_class_instances = dict(ClassInstance.objects.filter(project_id=self.target.id) \ .values_list('name', 'id')) existing_concept_ids = set(Concept.objects.all().values_list('id', flat=True)) # Find classes for neurons and skeletons in import data if Class in import_data: allowed_duplicate_classes = tuple(c.object.id for c in import_data.get(Class) if c.object.class_name in ('neuron', 'skeleton')) else: allowed_duplicate_classes = tuple() n_reused = 0 n_moved = 0 append_only = not self.preserve_ids need_separate_import = [] objects_to_save = defaultdict(list) import_objects_by_type_and_id = defaultdict(dict) for object_type, import_objects in six.iteritems(import_data): # Allow user reference updates in CATMAID objects if object_type not in user_updatable_classes: need_separate_import.append(object_type) continue # Stores in append-only mode import IDs and links them to the # respective objects. This is needed, to update foreign keys to this # ID when it is replaced with a new ID. objects_by_id = import_objects_by_type_and_id[object_type] is_class = object_type == Class is_relation = object_type == Relation is_class_instance = object_type == ClassInstance # CATMAID model objects are inspected for user fields for deserialized_object in import_objects: obj = deserialized_object.object # Semantic data like classes and class instances are expected to # be unique with respect to their names. Existing objects with # the same ID will get a new ID even if --preserve-ids is set. existing_obj_id = None concept_id_exists = obj.id in existing_concept_ids if is_class: existing_obj_id = existing_classes.get(obj.class_name) if is_relation: existing_obj_id = existing_relations.get(obj.relation_name) if is_class_instance: existing_obj_id = existing_class_instances.get(obj.name) # Neurons (class instances of class "neuron" and "skeleton") # are a special case. There can be multiple neurons with # the same name, something that is not allowed in other # cases. In this particular case, however, class instance # reuse is not wanted. if existing_obj_id and obj.class_column_id in allowed_duplicate_classes: existing_obj_id = None concept_id_exists = False if existing_obj_id is not None: # Add mapping so that existing references to it can be # updated. The object itself is not marked for saving, # because it exists already. current_id = obj.id objects_by_id[current_id] = obj obj.id = existing_obj_id n_reused += 1 continue # If there is already an known object with the ID of the object # we are importing at the moment and the current model is a # class, relation or class_instance, then the imported object # will get a new ID, even with --preservie-ids set. We reuse # these types. if concept_id_exists: current_id = obj.id objects_by_id[current_id] = obj obj.id = None n_moved += 1 # Replace existing data if requested self.override_fields(obj) # Map users based on username, optionally create unmapped users. self.map_or_create_users(obj, import_users, mapped_user_ids, mapped_user_target_ids, created_users) # Remove pre-defined ID and keep track of updated IDs in # append-only mode (default). if append_only: current_id = obj.id objects_by_id[current_id] = obj # By setting id to None, Django will create a new object and # set the new ID. obj.id = None # Remember for saving objects_to_save[object_type].append(deserialized_object) if len(created_users) > 0: logger.info("Created {} new users: {}".format(len(created_users), ", ".join(sorted(created_users.keys())))) else: logger.info("No unmapped users imported") # Finally save all objects. Make sure they are saved in order: logger.info("Storing {} database objects including {} moved objects, reusing additional {} existing objects" \ .format(n_objects - n_reused, n_moved, n_reused)) # In append-only mode, the foreign keys to objects with changed IDs have # to be updated. In preserve-ids mode only IDs to classes and relations # will be updated. Saving model objects after an update of referenced # keys is only needed in append-only mode. self.reset_ids(user_updatable_classes, objects_to_save, import_objects_by_type_and_id, existing_classes) other_tasks = set(objects_to_save.keys()) - set(ordered_save_tasks) for object_type in ordered_save_tasks + list(other_tasks): objects = objects_to_save.get(object_type) if objects: logger.info("- Importing objects of type " + object_type.__name__) for deserialized_object in progressbar.progressbar(objects, max_value=len(objects), redirect_stdout=True): deserialized_object.save() logger.info("- Importing all other objects") for other_model in progressbar.progressbar(need_separate_import, max_value=len(need_separate_import), redirect_stdout=True): other_objects = import_data[other_model] if other_model == User: # If user model objects are imported and users were mapped, ask # user if alrady mapped users should be skipped during import. # We don't need to take of newly created users, because they are # only created if no model is found. Therefore all other model # objects can be imported. if mapped_user_target_ids: mapped_usernames = set(self.user_id_map.get(u) for u in mapped_user_target_ids) import_usernames = set(import_users.keys()) not_imported_usernames = import_usernames - mapped_usernames already_imported_usernames = import_usernames - not_imported_usernames if already_imported_usernames: print("The following usernames are mapped to " + "existing users, but the import data " + "also contains objects for these users: " + ", ".join(already_imported_usernames)) ignore_users = ask_yes_no("Skip those users in input " "data and don't import them? [y/n]") if ignore_users: logger.info("Won't import mapped users: " + ", ".join(already_imported_usernames)) other_objects = [u for u in other_objects \ if u.object.username not in already_imported_usernames] else: logger.info("Will import all listed users in import data") for deserialized_object in other_objects: if deserialized_object.object.username in created_users.keys(): deserialized_object.save() # Reset counters to current maximum IDs cursor.execute(''' SELECT setval('concept_id_seq', coalesce(max("id"), 1), max("id") IS NOT null) FROM concept; SELECT setval('location_id_seq', coalesce(max("id"), 1), max("id") IS NOT null) FROM location; SELECT setval('auth_user_id_seq', coalesce(max("id"), 1), max("id") IS NOT null) FROM auth_user; ''') cursor.execute(""" CREATE TRIGGER on_insert_treenode_update_summary_and_edges AFTER INSERT ON treenode REFERENCING NEW TABLE as inserted_treenode FOR EACH STATEMENT EXECUTE PROCEDURE on_insert_treenode_update_summary_and_edges(); CREATE TRIGGER on_edit_treenode_update_summary_and_edges AFTER UPDATE ON treenode REFERENCING NEW TABLE as new_treenode OLD TABLE as old_treenode FOR EACH STATEMENT EXECUTE PROCEDURE on_edit_treenode_update_summary_and_edges(); CREATE TRIGGER on_delete_treenode_update_summary_and_edges AFTER DELETE ON treenode REFERENCING OLD TABLE as deleted_treenode FOR EACH STATEMENT EXECUTE PROCEDURE on_delete_treenode_update_summary_and_edges(); """) logger.info("Updating edge tables") rebuild_edge_tables(log=lambda msg: logger.info(msg)) logger.info("Updated skeleton summary tables") cursor.execute(""" DELETE FROM catmaid_skeleton_summary; SELECT refresh_skeleton_summary_table(); """)
def evaluate_coco(generator, model, threshold=0.05): """ Use the pycocotools to evaluate a COCO model on a dataset. Args generator : The generator for generating the evaluation data. model : The model to evaluate. threshold : The score threshold to use. """ # start collecting results results = [] image_ids = [] for index in progressbar.progressbar(range(generator.size()), prefix='COCO evaluation: '): image = generator.load_image(index) image = generator.preprocess_image(image) image, scale = generator.resize_image(image) if keras.backend.image_data_format() == 'channels_first': image = image.transpose((2, 0, 1)) # run network boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0)) # correct boxes for image scale boxes /= scale # change to (x, y, w, h) (MS COCO standard) boxes[:, :, 2] -= boxes[:, :, 0] boxes[:, :, 3] -= boxes[:, :, 1] # compute predicted labels and scores for box, score, label in zip(boxes[0], scores[0], labels[0]): # scores are sorted, so we can break if score < threshold: break # append detection for each positively labeled class image_result = { 'image_id' : generator.image_ids[index], 'category_id' : generator.label_to_coco_label(label), 'score' : float(score), 'bbox' : box.tolist(), } # append detection to results results.append(image_result) # append image to list of processed images image_ids.append(generator.image_ids[index]) if not len(results): return # write output json.dump(results, open('{}_bbox_results.json'.format(generator.set_name), 'w'), indent=4) json.dump(image_ids, open('{}_processed_image_ids.json'.format(generator.set_name), 'w'), indent=4) # load results in COCO evaluation tool coco_true = generator.coco coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(generator.set_name)) # run COCO evaluation coco_eval = COCOeval(coco_true, coco_pred, 'bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return coco_eval.stats