def create_lmdb(paths_labels, lmdb_save_path): # configurations # img_folder = 'D:/dataloader/RSSCN7/train_lr/*' # glob matching pattern # lmdb_save_path = 'D:/dataloader/RSSCN7/rsscn7_bicLRx4.lmdb' # must end with .lmdb dataset = [] data_size = 0 print('Read images...') pbar = ProgressBar(len(paths_labels)) for i, (v, l) in enumerate(paths_labels): img = cv2.imread(v, cv2.IMREAD_UNCHANGED) dataset.append(img) data_size += img.nbytes # pbar.update('Read {}'.format(v)) env = lmdb.open(lmdb_save_path, map_size=data_size * 10) print('Finish reading {} images.\nWrite lmdb...'.format(len(paths_labels))) pbar = ProgressBar(len(paths_labels)) with env.begin(write=True) as txn: # txn is a Transaction object for i, (v, l) in enumerate(paths_labels): base_name = os.path.splitext(os.path.basename(v))[0] key = base_name.encode('ascii') data = dataset[i] if dataset[i].ndim == 2: H, W = dataset[i].shape C = 1 else: H, W, C = dataset[i].shape meta_key = (base_name + '.meta').encode('ascii') meta = '{:d}, {:d}, {:d}'.format(H, W, C) label_key = (base_name + '.label').encode('ascii') label = '{:d}'.format(l) # The encode is only essential in Python 3 txn.put(key, data) txn.put(meta_key, meta.encode('ascii')) txn.put(label_key, label.encode('ascii')) # pbar.update('Write {}'.format(v)) print('Finish writing lmdb.') # create keys cache keys_cache_file = os.path.join(lmdb_save_path, '_keys_cache.p') env = lmdb.open(lmdb_save_path, readonly=True, lock=False, readahead=False, meminit=False) with env.begin(write=False) as txn: print('Create lmdb keys cache: {}'.format(keys_cache_file)) keys = [key.decode('ascii') for key, _ in txn.cursor()] pickle.dump(keys, open(keys_cache_file, "wb")) print('Finish creating lmdb keys cache.')
def __remove_colliders(self): pbar = ProgressBar(list(self.colliders.values()), title="REMOVE TILE COLLIDERS") for guid, collider in self.colliders.items(): self.objects_xml.remove_object(guid) pbar.update("%s removed" % collider.name) self.__clean_objects(self.colliders)
def query_search(url): sublist = [] engine_urls = [ 'https://www.virustotal.com/en/domain/%s/information/', 'https://www.google.com/search?q=site:%s&num=100', 'https://search.yahoo.com/search?p=%s&b=1' ] i = 0 for engine in engine_urls: request_handler = RequestHandler() data = request_handler.send(engine % url.replace("http://", "")) if data == 'None': return None soup = BeautifulSoup(data, "lxml") results = set( re.findall(r"\w+\.{}".format(url.replace("http://", "")), soup.text)) for subdomain in results: if "www." not in subdomain: output = GREEN + "%s\n" % (subdomain) + RESET sublist.append(output) i += 1 progressBar = ProgressBar() progressBar.progress(i, len(engine_urls), status='Searching') return sublist
def _test_step(model, optimizer, batch_dim, test_batch, start_time, _test_update): self.load(directory) from_start = timedelta(seconds=int((time.time() - start_time))) self.log('End of training ({} epochs) in {}'.format( epochs, from_start)) if test_batch is not None: pr = ProgressBar(80, test_batch) output = defaultdict(list) for i in range(test_batch): for k, v in self.session.run(test_fetch_dict( model, optimizer), feed_dict=test_feed_dict( model, optimizer, batch_dim)).items(): output[k].append(v) pr.update(i + 1) self.log(date=False) output = {k: np.mean(v) for k, v in output.items()} else: output = self.session.run( test_fetch_dict(model, optimizer), feed_dict=test_feed_dict(model, optimizer, batch_dim)) if _test_update is not None: output.update( _test_update(model, optimizer, batch_dim, test_batch)) p = pprint.PrettyPrinter(indent=1, width=80) self.log('Test --> {}'.format(p.pformat(output))) for k in output: self.print['Test ' + k].append(output[k]) return output
def __retrieve_scene_objects(self): pbar = ProgressBar(list( Path(self.model_lib_folder).rglob(XML_FILE_PATTERN)), title="Retrieve project infos") for i, path in enumerate(pbar.iterable): if not is_octant(path.stem): msfs_scene_object = MsfsSceneObject(self.model_lib_folder, path.stem, path.name) self.objects[msfs_scene_object.xml.guid] = msfs_scene_object pbar.update("%s" % path.name) continue if COLLIDER_SUFFIX in path.stem: msfs_collider = MsfsCollider(self.model_lib_folder, path.stem, path.name) self.colliders[msfs_collider.xml.guid] = msfs_collider pbar.update("%s" % path.name) continue msfs_tile = MsfsTile(self.model_lib_folder, path.stem, path.name) if not msfs_tile.lods: msfs_tile.remove_files() else: self.tiles[msfs_tile.xml.guid] = msfs_tile pbar.update("%s" % path.name)
def main(input_folder, save_folder): input_folder += str(config["upscale_factor"]) save_folder += str(config['upscale_factor']) """A multi-thread tool to crop sub imags.""" n_thread = 20 compression_level = 0 # 3 is the default value in cv2 # CV_IMWRITE_PNG_COMPRESSION from 0 to 9. A higher value means a smaller size and longer # compression time. If read raw images during training, use 0 for faster IO speed. if not os.path.exists(save_folder): os.makedirs(save_folder) print('mkdir [{:s}] ...'.format(save_folder)) else: print('Folder [{:s}] already exists. Exit...'.format(save_folder)) img_list = [] for root, _, file_list in sorted(os.walk(input_folder)): path = [os.path.join(root, x) for x in file_list] # assume only images in the input_folder img_list.extend(path) def update(arg): pbar.update(arg) pbar = ProgressBar(len(img_list)) print("Start 2 ...") pool = Pool(n_thread) for path in img_list: pool.apply_async(worker, args=(path, save_folder, compression_level), callback=update) pool.close() pool.join() print('All subprocesses done.')
def extract_feat_faster_start(args, cfg): os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id num_gpus = len(args.gpu_id.split(',')) MIN_BOXES = cfg.MODEL.BUA.EXTRACTOR.MIN_BOXES MAX_BOXES = cfg.MODEL.BUA.EXTRACTOR.MAX_BOXES CONF_THRESH = cfg.MODEL.BUA.EXTRACTOR.CONF_THRESH # Extract features. imglist = os.listdir(args.image_dir) num_images = len(imglist) print('Number of images: {}.'.format(num_images)) if args.num_cpus != 0: ray.init(num_cpus=args.num_cpus) else: ray.init() img_lists = [imglist[i::num_gpus] for i in range(num_gpus)] pb = ProgressBar(len(imglist)) actor = pb.actor print('Number of GPUs: {}.'.format(num_gpus)) extract_feat_list = [] for i in range(num_gpus): extract_feat_list.append( extract_feat_faster.remote(i, img_lists[i], cfg, args, actor)) pb.print_until_done() ray.get(extract_feat_list) ray.get(actor.get_counter.remote())
def split_tiles(self): self.__split_tiles(self.__retrieve_tiles_to_process()) previous_tiles = {guid: tile for guid, tile in self.tiles.items()} new_tiles = {} # reload the project to retrieve the new tiles self.__retrieve_scene_objects() # create the matching dictionary between the previous tiles and the corresponding splitted tiles for previous_guid, previous_tile in previous_tiles.items(): new_tiles[previous_tile] = [ tile for tile in self.tiles.values() if previous_tile.name in tile.name and previous_tile.name != tile.name ] pbar = ProgressBar( new_tiles.items(), title= "REPLACE THE OLD TILES BY THE NEW SPLITTED TILES IN THE SCENE DEFINITION FILE" ) for previous_tile, new_tiles in new_tiles.items(): self.__replace_tiles_in_objects_xml(previous_tile, new_tiles) pbar.update( "splitted tiles added, replacing the previous %s tile" % previous_tile.name)
def __convert_dds_texture_files(self): ON_POSIX = 'posix' in sys.builtin_module_names self.converted_data, data = itertools.tee(self.converted_data) pbar = ProgressBar(list(data), title="convert " + self.DDS_FORMAT + " textures to " + self.BMP_FORMAT) for chunck in self.converted_data: # create a pipe to get data input_fd, output_fd = os.pipe() for obj in chunck: print("-------------------------------------------------------------------------------") print("prepare command line: ", self.path, obj['file_name'], obj['dest_file_name']) processes = [subprocess.Popen([self.path, obj['file_name'], obj['dest_file_name']], stdout=output_fd, close_fds=ON_POSIX) for obj in chunck] os.close(output_fd) # close unused end of the pipe # read output line by line as soon as it is available with io.open(input_fd, "r") as file: for line in file: print(line, end=str()) for p in processes: p.wait() pbar.update("%s converted to %s" % (os.path.basename(obj['file_name']), os.path.basename(obj['dest_file_name'])))
def run(self, data): results_file_name = "results/" + re.sub(".*/", "", data) + "_email_found.txt" data = pd.read_csv(data, sep="\t") colnames = list(data.columns) + ["邮箱", "作者信息", "作者机构", "文章标题", "查找方式"] data = data.values try: print("results stored in {}".format(results_file_name)) done = self.__net.get_file_column(results_file_name) done = [list(i) for i in done] except Exception as e: print(e) print("did not found results table, create a new one") done = [] bar = ProgressBar(data.shape[0]) with open(results_file_name, mode='a', encoding='utf-8') as results: if not len(done): results.write('\t'.join(colnames) + '\n') for row in data: row = [str(i) for i in row] author, organ = (row[0].strip(), row[1].strip()) if not self.array_in([author, organ], done): email = self.iter_find_author_email(author, organ) results.write('\t'.join(list(row) + email) + '\n') done.append([author, organ]) results.flush() bar.move() else: bar.move()
def fix_tiles_lightning_issues(self, settings): isolated_print(EOL) lods = [lod for tile in self.tiles.values() for lod in tile.lods] pbar = ProgressBar(list(lods), title="FIX TILES LIGHTNING ISSUES") for lod in lods: lod.optimization_in_progress = False lod.prepare_for_msfs() pbar.update("%s lightning issues fixed" % lod.name)
def __backup_objects(objects: dict, backup_path, pbar_title="backup files"): pbar = ProgressBar(list()) for guid, object in objects.items(): object.backup_files(backup_path, dry_mode=True, pbar=pbar) if pbar.range > 0: pbar.display_title(pbar_title) for guid, object in objects.items(): object.backup_files(backup_path, pbar=pbar)
def update_min_size_values(self, settings): pbar = ProgressBar(list()) pbar.range = len(self.tiles) + len(self.colliders) pbar.display_title("Update lod values") for tile in self.tiles.values(): tile.update_min_size_values(settings.target_min_size_values, pbar=pbar) for collider in self.colliders.values(): collider.update_min_size_values(settings.target_min_size_values, pbar=pbar)
def fix_object_bounding_box(resize_box=True): if not bpy.context.scene: return create_collection = bpy.data.collections.new(name=COPY_COLLECTION_NAME) bpy.context.scene.collection.children.link(create_collection) assert (create_collection is not bpy.context.scene.collection) # copy objects copy_objects(bpy.context.scene.collection, create_collection, False) obs = [] for obj in create_collection.objects: if obj.type == MESH_OBJECT_TYPE: obs.append(obj) ctx = bpy.context.copy() if len(obs) < 1: return None ctx[ACTIVE_OBJ] = obs[0] ctx[SELECTED_OBJ] = obs # In Blender 2.8x this needs to be the following instead: ctx[SELECTED_EDITABLE_OBJ] = obs # join copied objects bpy.ops.object.join(ctx) bpy.ops.object.select_all(action=SELECT_ACTION) objects = bpy.context.scene.objects # fix objects origin: this also fixes the bounding box for the whole tile pbar = ProgressBar(objects) for obj in objects: center_origin(obj) pbar.update("bounded box updated for %s" % obj.name) bpy.ops.object.select_all(action=DESELECT_ACTION) # remove joined copied objects for obj in create_collection.objects: obj.select_set(True) bpy.ops.object.delete() bpy.ops.object.select_all(action=SELECT_ACTION) for c in bpy.context.scene.collection.children: bpy.context.scene.collection.children.unlink(c) if resize_box: # resize objects to fix spacing between tiles bpy.ops.transform.resize(value=(1.0045, 1.0045, 1))
def __multithread_process_data(processed_data, script_name, title, update_msg): ON_POSIX = 'posix' in sys.builtin_module_names processed_data, data = itertools.tee(processed_data) pbar = ProgressBar(list(data), title=title) try: for chunck in processed_data: # create a pipe to get data input_fd, output_fd = os.pipe() params = [ str(bpy.app.binary_path), "--background", "--python", os.path.join(os.path.dirname(os.path.dirname(__file__)), script_name), "--" ] for obj in chunck: print( "-------------------------------------------------------------------------------" ) print( "prepare command line: ", "\"" + str(bpy.app.binary_path) + "\" --background --python \"" + os.path.join( os.path.dirname(os.path.dirname(__file__)), script_name) + "\" -- " + str(" ").join(obj["params"])) si = subprocess.STARTUPINFO() si.dwFlags = subprocess.STARTF_USESTDHANDLES | subprocess.HIGH_PRIORITY_CLASS processes = [ subprocess.Popen(params + obj["params"], stdout=output_fd, stderr=subprocess.DEVNULL, close_fds=ON_POSIX, startupinfo=si, encoding=ENCODING) for obj in chunck ] os.close(output_fd) # close unused end of the pipe # read output line by line as soon as it is available with io.open(input_fd, "r", buffering=1) as file: for line in file: print(line, end=str()) for p in processes: p.wait() pbar.update("%s %s" % (obj["name"], update_msg)) except: pass
def __retrieve_shapes(self): pbar = ProgressBar(list( Path(self.scene_folder).rglob(DBF_FILE_PATTERN)), title="Retrieve shapes") for i, path in enumerate(pbar.iterable): self.shapes[path.stem] = MsfsShape(self.scene_folder, path.stem, path.stem + XML_FILE_EXT, path.name, path.stem + SHP_FILE_EXT, path.stem + SHX_FILE_EXT) pbar.update("%s" % path.name)
def run(self, journal_list_file): base_path = os.path.dirname(os.path.abspath(__file__)) + '/' results_file_name = base_path + "results/" + \ os.path.splitext(os.path.basename(journal_list_file))[ 0] + "_article_information.txt" done_journal_list_file_name = base_path + "data/done_journals.txt" with open(journal_list_file) as jl: journal_list = [ jn.strip() for jn in jl.read().split('\n') if jn.strip() ] colnames = [ '文章ID', '期刊名', '年份', '期', '论文题目', '作者', '机构', '关键词', '作者简介', '通讯作者姓名', '通讯作者邮箱', '通讯作者电话', '第一作者姓名', '第一作者邮箱', '第一作者电话' ] try: print("results stored in {}".format(results_file_name)) done = self.__net.get_file_column(results_file_name, number=1) done = done.tolist() except Exception: # print(e) print("did not found results table, create a new one") done = [] try: with open(done_journal_list_file_name) as djl: done_journal_list = [ jn.strip() for jn in djl.read().split('\n') if jn.strip() ] except Exception: done_journal_list = [] bar = ProgressBar(len(journal_list)) with open(results_file_name, mode='a', encoding='utf-8') as results, open( done_journal_list_file_name, mode='a', encoding='utf-8') as done_journal_file: if not len(done): results.write('\t'.join(colnames) + '\n') for journal in journal_list: if journal not in done_journal_list: for paper_id in self.all_journal_ids(journal): if paper_id not in done: found_info = self.find_article_info(paper_id) results.write('\t'.join(found_info) + '\n') done.append(paper_id) # results.flush() done_journal_file.write(journal + '\n') done_journal_list.append(journal) # done_journal_file.flush() bar.move() else: bar.move()
def add_tile_colliders(self): # clean previous colliders self.__remove_colliders() lods = [lod for tile in self.tiles.values() for lod in tile.lods] pbar = ProgressBar( list(lods), title="REMOVE ROAD AND COLLISION TAGS IN THE TILE LODS") for lod in lods: lod.optimization_in_progress = False lod.remove_road_and_collision_tags() pbar.update("road and collision tags removed from %s" % lod.name) pbar = ProgressBar(list(self.tiles.values()), title="ADD TILE COLLIDERS") for tile in self.tiles.values(): tile_guid = tile.xml.guid new_collider = tile.add_collider() self.__add_object_in_objects_xml(tile_guid, new_collider) pbar.update("collider added for %s tile" % tile.name)
def remove_colliders(self): # clean previous colliders self.__remove_colliders() lods = [lod for tile in self.tiles.values() for lod in tile.lods] pbar = ProgressBar( list(lods), title="ADD ROAD AND COLLISION TAGS IN THE TILE LODS") for lod in lods: lod.optimization_in_progress = False lod.prepare_for_msfs() pbar.update("road and collision tags added from %s" % lod.name)
def backup_files(self, backup_subfolder): backup_path = os.path.join(self.backup_folder, backup_subfolder) if not os.path.isfile( get_backup_file_path(backup_path, self.scene_folder, self.SCENE_OBJECTS_FILE)): pbar = ProgressBar([self.SCENE_OBJECTS_FILE], title="backup " + self.SCENE_OBJECTS_FILE) backup_file(backup_path, self.scene_folder, self.SCENE_OBJECTS_FILE, pbar=pbar)
def _generate_AX(self): self.log('Creating features and adjacency matrices..') pr = ProgressBar(60, len(self.data)) data = [] smiles = [] data_S = [] data_A = [] data_X = [] data_D = [] data_F = [] data_Le = [] data_Lv = [] max_length = max(mol.GetNumAtoms() for mol in self.data) max_length_s = max(len(Chem.MolToSmiles(mol)) for mol in self.data) for i, mol in enumerate(self.data): A = self._genA(mol, connected=True, max_length=max_length) D = np.count_nonzero(A, -1) if A is not None: data.append(mol) smiles.append(Chem.MolToSmiles(mol)) data_S.append(self._genS(mol, max_length=max_length_s)) data_A.append(A) data_X.append(self._genX(mol, max_length=max_length)) data_D.append(D) data_F.append(self._genF(mol, max_length=max_length)) L = D - A Le, Lv = np.linalg.eigh(L) data_Le.append(Le) data_Lv.append(Lv) pr.update(i + 1) self.log(date=False) self.log( 'Created {} features and adjacency matrices out of {} molecules!'. format(len(data), len(self.data))) self.data = data self.smiles = smiles self.data_S = data_S self.data_A = data_A self.data_X = data_X self.data_D = data_D self.data_F = data_F self.data_Le = data_Le self.data_Lv = data_Lv self.__len = len(self.data)
def main(): parser = argparse.ArgumentParser( 'Extract sub images from Hyperspectral images.') parser.add_argument('-p', '--path', type=str, default='E:/HSI/CAVE/CAVE/') parser.add_argument('-o', '--out-path', type=str, default='') parser.add_argument('-t', '--threads', type=int, default=1) parser.add_argument('-c', '--crop-size', type=int, default=256) parser.add_argument('-s', '--stride', type=int, default=96) opt = parser.parse_args() in_path = opt.path out_path = opt.out_path n_threads = opt.threads crop_sz = opt.crop_size stride = opt.stride thres_sz = 30 # compression_level = 3 if out_path == '': dataset_name = in_path.split('/')[-2].split('\\')[-1] if in_path[-1] == '/': out_path = in_path.replace(dataset_name + '/', dataset_name + '_sub.h5') else: out_path = in_path.replace(dataset_name, dataset_name + '_sub.h5') # if not os.path.exists(out_path): # os.makedirs(out_path) # print('mkdir [{:s}] ...'.format(out_path)) # else: # print('[*] Folder [{:s}] already exists.'.format(out_path)) # # return img_list = [] for root, _, file_list in sorted(os.walk(in_path)): path = [os.path.join(root, x) for x in file_list] img_list.extend(path) def update(arg): pbar.update(arg) pbar = ProgressBar(len(img_list)) pool = Pool(n_threads) for path in img_list: pool.apply_async(worker, args=(path, out_path, crop_sz, stride, thres_sz, compression_level), callback=update) pool.close() pool.join() print("-----------Generation Finish-------------")
def __create_optimization_folders(self): pbar = ProgressBar(list()) link_tiles_by_position = self.__link_tiles_by_position() for parent_tile, tiles in link_tiles_by_position.items(): parent_tile.create_optimization_folders(tiles, dry_mode=True, pbar=pbar) if pbar.range > 0: pbar.display_title("Create optimization folders") for parent_tile, tiles in link_tiles_by_position.items(): parent_tile.create_optimization_folders(tiles, dry_mode=False, pbar=pbar)
def drawResultDirectory(filePath, resultPath, *args): fileList = sorted(glob(filePath)) pbar = ProgressBar(len(fileList)) for index, value in enumerate(fileList): pbar.update("Read {}".format(value)) # Read image img = Image.open(value, "r") img = drawResultFile(img, args) img.save(resultPath + os.path.basename(value)) print("Completed!")
def _eval_step(epoch, epochs, min_epochs, model, optimizer, batch_dim, eval_batch, start_time, last_epoch_start_time, _eval_update): from_start = timedelta(seconds=int((time.time() - start_time))) last_epoch = timedelta(seconds=int((time.time() - last_epoch_start_time))) eta = timedelta( seconds=int((time.time() - start_time) * (epochs - epoch) / epoch)) if (time.time() - start_time) > 1 else '-:--:-' self.log( 'Epochs {:10}/{} in {} (last epoch in {}), ETA: {}'.format( epoch, epochs, from_start, last_epoch, eta)) if eval_batch is not None: pr = ProgressBar(80, eval_batch) output = defaultdict(list) for i in range(eval_batch): for k, v in self.session.run( eval_fetch_dict(epoch, epochs, min_epochs, model, optimizer), feed_dict=eval_feed_dict( epoch, epochs, min_epochs, model, optimizer, batch_dim)).items(): output[k].append(v) pr.update(i + 1) self.log(date=False) output = {k: np.mean(v) for k, v in output.items()} else: output = self.session.run( eval_fetch_dict(epoch, epochs, min_epochs, model, optimizer), feed_dict=eval_feed_dict(epoch, epochs, min_epochs, model, optimizer, batch_dim)) if _eval_update is not None: output.update( _eval_update(epoch, epochs, min_epochs, model, optimizer, batch_dim, eval_batch)) p = pprint.PrettyPrinter(indent=1, width=80) self.log('Validation --> {}'.format(p.pformat(output))) for k in output: self.print[k].append(output[k]) return output
def __find_different_tiles(self, tiles, project_to_compare_name, tiles_to_compare, objects_xml_to_compare): different_tiles = [] pbar = ProgressBar(tiles.items(), title="FIND THE DIFFERENT TILES") for guid, tile in pbar.iterable: found_tile = self.__find_by_tile_name(tile, tiles_to_compare) if not found_tile: different_tiles.append(tile) elif len(tile.lods) != len(found_tile.lods): different_tiles.append(tile) pbar.update("%s checked" % tile.name) return different_tiles
def _genReward(self, metric, batch_size=15): self.log('Calculating molecule rewards..') pr = ProgressBar(60, len(self.data)) i = 0 self.data_rwd = [] while i < len(self.data): mols = self.data[i:i + batch_size] rwds = reward(mols, metric, self).reshape(-1) self.data_rwd.append(rwds) i += batch_size pr.update(min(i, len(self.data))) self.data_rwd = np.concatenate(self.data_rwd, -1)
def download_file(v_data, directory): if not v_data: return False file_path = directory + v_data["name"] if Path(file_path).exists(): print(f"Already exists: ", end="") return True r = requests.get(v_data["url"], stream=True) indicator = ProgressBar(int(r.headers.get('content-length'))) with open(file_path, 'wb') as f: for chunk in r.iter_content(chunk_size=1024 * 1024): if chunk: f.write(chunk) indicator.progress += len(chunk) return True
def val(model, criterion, dataloader, epoch=None, val_writer=None, lr=None, msglogger=None): with t.no_grad(): """ 计算模型在验证集上的准确率等信息 """ model.eval() val_losses = AverageMeter() val_top1 = AverageMeter() val_top5 = AverageMeter() val_progressor = None if not msglogger: val_progressor = ProgressBar(mode="Val ", epoch=epoch, total_epoch=opt.max_epoch, model_name=opt.model, lr=lr, total=len(dataloader)) for ii, (data, labels, img_path,tag) in enumerate(dataloader): if not check_date(img_path, tag, msglogger): return input = data.to(opt.device) labels = labels.to(opt.device) score = model(input) loss = criterion(score, labels) precision1, precision5 = accuracy(score, labels, topk=(1, 5)) # top1 和 top5 的准确率 val_losses.update(loss.item(), input.size(0)) val_top1.update(precision1[0].item(), input.size(0)) val_top5.update(precision5[0].item(), input.size(0)) if val_progressor: val_progressor.current = ii + 1 val_progressor.current_loss = val_losses.avg val_progressor.current_top1 = val_top1.avg val_progressor.current_top5 = val_top5.avg val_progressor() if ii % opt.print_freq == 0: if val_writer: grid = make_grid((input.data.cpu() * 0.225 + 0.45).clamp(min=0, max=1)) val_writer.add_image('val_images', grid, ii * (epoch + 1)) # 测试图片 val_writer.add_scalar('loss', val_losses.avg, ii * (epoch + 1)) # 训练误差 val_writer.add_text('top1', 'val accuracy top1 %.2f%%' % val_top1.avg, ii * (epoch + 1)) # top1准确率文本 val_writer.add_scalars('accuracy', {'top1': val_top1.avg, 'top5': val_top5.avg, 'loss': val_losses.avg}, ii * (epoch + 1)) if msglogger: msglogger.info('==> Top1: %.3f Top5: %.3f Loss: %.3f\n', val_top1.avg, val_top5.avg, val_losses.avg) return [val_losses.avg, val_top1.avg, val_top5.avg]
def main(): """A multi-thread tool to crop sub imags.""" dataset_root_path = '/home/ybahat/Datasets' if gethostname( ) == 'ybahat-System-Product-Name' else '/home/tiras/datasets' if 'tiras' in os.getcwd( ) else '/media/ybahat/data/Datasets' input_folder = os.path.join(dataset_root_path, 'DIV2K_train/DIV2K_train_HR') save_folder = os.path.join(dataset_root_path, 'DIV2K_train/DIV2K_train_sub_HR') n_thread = 20 crop_sz = 256 #480 step = 30 #240 thres_sz = 48 compression_level = 3 # 3 is the default value in cv2 multi_scale = False # CV_IMWRITE_PNG_COMPRESSION from 0 to 9. A higher value means a smaller size and longer # compression time. If read raw images during training, use 0 for faster IO speed. if not os.path.exists(save_folder): os.makedirs(save_folder) print('mkdir [{:s}] ...'.format(save_folder)) else: print('Folder [{:s}] already exists. Exit...'.format(save_folder)) sys.exit(1) img_list = [] for root, _, file_list in sorted(os.walk(input_folder)): path = [os.path.join(root, x) for x in file_list] # assume only images in the input_folder img_list.extend(path) def update(arg): pbar.update(arg) pbar = ProgressBar(len(img_list)) pool = Pool(n_thread) for path in img_list: pool.apply_async(worker, args=(path, save_folder, crop_sz, step, thres_sz, compression_level, multi_scale), callback=update) pool.close() pool.join() print('All subprocesses done.')