def export_tagged_chips(ibs, aid_list, dpath='.'): """ DEPRICATE CommandLine: python -m wbia.tag_funcs --exec-export_tagged_chips --tags Hard interesting needswork --db PZ_Master1 python -m wbia.tag_funcs --exec-export_tagged_chips --logic=or --any_startswith quality occlusion --has_any lighting needswork interesting hard --db GZ_Master1 --dpath=/media/raid python -m wbia.tag_funcs --exec-export_tagged_chips --db GZ_Master1 --min_num=1 --dpath /media/raid Example: >>> # SCRIPT >>> from wbia.tag_funcs import * # NOQA >>> import wbia >>> ibs = wbia.opendb(defaultdb='testdb1') >>> kwargs = ut.argparse_dict(ut.get_kwdefaults2(filterflags_general_tags), type_hint=ut.ddict(list, logic=str)) >>> ut.print_dict(kwargs, 'filter args') >>> aid_list = ibs.filter_annots_by_tags(**kwargs) >>> print('len(aid_list) = %r' % (len(aid_list),)) >>> dpath = ut.get_argval('--dpath', default='') >>> all_tags = ut.flatten(ibs.get_annot_all_tags(aid_list)) >>> filtered_tag_hist = ut.dict_hist(all_tags) >>> ut.print_dict(filtered_tag_hist, key_order_metric='val') >>> export_tagged_chips(ibs, aid_list, dpath) """ visual_uuid_hashid = ibs.get_annot_hashid_visual_uuid(aid_list) zip_fpath = ut.unixjoin( dpath, 'exported_chips2_' + ibs.get_dbname() + visual_uuid_hashid + '.zip') chip_fpath = ibs.get_annot_chip_fpath(aid_list) ut.archive_files(zip_fpath, chip_fpath, common_prefix=True)
def export_tagged_chips(ibs, aid_list, dpath='.'): """ CommandLine: python -m ibeis.tag_funcs --exec-export_tagged_chips --tags Hard interesting needswork --db PZ_Master1 python -m ibeis.tag_funcs --exec-export_tagged_chips --logic=or --any_startswith quality occlusion --has_any lighting needswork interesting hard --db GZ_Master1 --dpath=/media/raid python -m ibeis.tag_funcs --exec-export_tagged_chips --db GZ_Master1 --min_num=1 --dpath /media/raid Example: >>> # SCRIPT >>> from ibeis.tag_funcs import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb(defaultdb='testdb1') >>> kwargs = ut.argparse_dict(ut.get_kwdefaults2(filterflags_general_tags), type_hint=ut.ddict(list, logic=str)) >>> ut.print_dict(kwargs, 'filter args') >>> aid_list = ibs.filter_annots_by_tags(**kwargs) >>> print('len(aid_list) = %r' % (len(aid_list),)) >>> dpath = ut.get_argval('--dpath', default='') >>> all_tags = ut.flatten(ibs.get_annot_all_tags(aid_list)) >>> filtered_tag_hist = ut.dict_hist(all_tags) >>> ut.print_dict(filtered_tag_hist, key_order_metric='val') >>> export_tagged_chips(ibs, aid_list, dpath) """ visual_uuid_hashid = ibs.get_annot_hashid_visual_uuid(aid_list, _new=True) zip_fpath = ut.unixjoin(dpath, 'exported_chips2_' + ibs.get_dbname() + visual_uuid_hashid + '.zip') chip_fpath = ibs.get_annot_chip_fpath(aid_list) ut.archive_files(zip_fpath, chip_fpath, common_prefix=True)
def labeler_train(ibs, species_list=None, species_mapping=None, viewpoint_mapping=None, ensembles=3, **kwargs): from ibeis_cnn.ingest_ibeis import get_cnn_labeler_training_images_pytorch from ibeis.algo.detect import densenet species = '-'.join(species_list) args = (species, ) data_path = join(ibs.get_cachedir(), 'extracted-labeler-%s' % args) extracted_path = get_cnn_labeler_training_images_pytorch( ibs, category_list=species_list, category_mapping=species_mapping, viewpoint_mapping=viewpoint_mapping, dest_path=data_path, **kwargs) weights_path_list = [] for ensemble_num in range(ensembles): args = ( species, ensemble_num, ) output_path = join(ibs.get_cachedir(), 'training', 'labeler-%s-ensemble-%d' % args) if exists(output_path): ut.delete(output_path) weights_path = densenet.train(extracted_path, output_path, blur=False, flip=False) weights_path_list.append(weights_path) args = (species, ) output_name = 'labeler.%s' % args ensemble_path = join(ibs.get_cachedir(), 'training', output_name) ut.ensuredir(ensemble_path) archive_path = '%s.zip' % (ensemble_path) ensemble_weights_path_list = [] for index, weights_path in enumerate(sorted(weights_path_list)): assert exists(weights_path) ensemble_weights_path = join(ensemble_path, 'labeler.%d.weights' % (index, )) ut.copy(weights_path, ensemble_weights_path) ensemble_weights_path_list.append(ensemble_weights_path) ensemble_weights_path_list = [ensemble_path] + ensemble_weights_path_list ut.archive_files(archive_path, ensemble_weights_path_list, overwrite=True, common_prefix=True) return archive_path
def canonical_classifier_train(ibs, species, ensembles=3, extracted_path=None, **kwargs): from wbia.other.detectexport import ( get_cnn_classifier_canonical_training_images_pytorch, ) from wbia.algo.detect import densenet args = (species, ) data_path = join(ibs.get_cachedir(), 'extracted-classifier-canonical-%s' % args) if extracted_path is None: extracted_path = get_cnn_classifier_canonical_training_images_pytorch( ibs, species, dest_path=data_path, ) weights_path_list = [] for ensemble_num in range(ensembles): args = ( species, ensemble_num, ) output_path = join(ibs.get_cachedir(), 'training', 'classifier-canonical-%s-ensemble-%d' % args) if exists(output_path): ut.delete(output_path) weights_path = densenet.train(extracted_path, output_path, blur=False, flip=False) weights_path_list.append(weights_path) args = (species, ) output_name = 'classifier.canonical.%s' % args ensemble_path = join(ibs.get_cachedir(), 'training', output_name) ut.ensuredir(ensemble_path) archive_path = '%s.zip' % (ensemble_path) ensemble_weights_path_list = [] for index, weights_path in enumerate(sorted(weights_path_list)): assert exists(weights_path) ensemble_weights_path = join( ensemble_path, 'classifier.canonical.%d.weights' % (index, )) ut.copy(weights_path, ensemble_weights_path) ensemble_weights_path_list.append(ensemble_weights_path) ensemble_weights_path_list = [ensemble_path] + ensemble_weights_path_list ut.archive_files(archive_path, ensemble_weights_path_list, overwrite=True, common_prefix=True) return archive_path
def archive(dstcnvs_normer, cachedir=None, overwrite=False): cachedir = dstcnvs_normer.cachedir if cachedir is None else cachedir data_fpath = dstcnvs_normer.get_fpath(cachedir) # flann_fpath = dstcnvs_normer.get_flann_fpath(cachedir) archive_fpath = dstcnvs_normer.get_fpath(cachedir, ext='.zip') fpath_list = [ data_fpath, # flann_fpath ] ut.archive_files(archive_fpath, fpath_list, overwrite=overwrite) return archive_fpath
def archive(dstcnvs_normer, cachedir=None, overwrite=False): cachedir = dstcnvs_normer.cachedir if cachedir is None else cachedir data_fpath = dstcnvs_normer.get_fpath(cachedir) #flann_fpath = dstcnvs_normer.get_flann_fpath(cachedir) archive_fpath = dstcnvs_normer.get_fpath(cachedir, ext='.zip') fpath_list = [ data_fpath, #flann_fpath ] ut.archive_files(archive_fpath, fpath_list, overwrite=overwrite) return archive_fpath
def canonical_localizer_train(ibs, species, ensembles=3, **kwargs): from ibeis_cnn.ingest_ibeis import get_cnn_localizer_canonical_training_images_pytorch from ibeis.algo.detect import canonical args = (species, ) data_path = join(ibs.get_cachedir(), 'extracted-localizer-canonical-%s' % args) extracted_path = get_cnn_localizer_canonical_training_images_pytorch( ibs, species, dest_path=data_path, ) weights_path_list = [] for ensemble_num in range(ensembles): args = ( species, ensemble_num, ) output_path = join(ibs.get_cachedir(), 'training', 'localizer-canonical-%s-ensemble-%d' % args) weights_path = canonical.train(extracted_path, output_path) weights_path_list.append(weights_path) args = (species, ) output_name = 'localizer.canonical.%s' % args ensemble_path = join(ibs.get_cachedir(), 'training', output_name) ut.ensuredir(ensemble_path) archive_path = '%s.zip' % (ensemble_path) ensemble_weights_path_list = [] for index, weights_path in enumerate(sorted(weights_path_list)): assert exists(weights_path) ensemble_weights_path = join( ensemble_path, 'localizer.canonical.%d.weights' % (index, )) ut.copy(weights_path, ensemble_weights_path) ensemble_weights_path_list.append(ensemble_weights_path) ensemble_weights_path_list = [ensemble_path] + ensemble_weights_path_list ut.archive_files(archive_path, ensemble_weights_path_list, overwrite=True, common_prefix=True) return archive_path
def classifier_cameratrap_densenet_train(ibs, positive_imageset_id, negative_imageset_id, ensembles=3, **kwargs): from wbia.other.detectexport import ( get_cnn_classifier_cameratrap_binary_training_images_pytorch, ) from wbia.algo.detect import densenet data_path = join(ibs.get_cachedir(), 'extracted-classifier-cameratrap') extracted_path = get_cnn_classifier_cameratrap_binary_training_images_pytorch( ibs, positive_imageset_id, negative_imageset_id, dest_path=data_path, image_size=densenet.INPUT_SIZE, **kwargs, ) weights_path_list = [] for ensemble_num in range(ensembles): args = (ensemble_num, ) output_path = join(ibs.get_cachedir(), 'training', 'classifier-cameratrap-ensemble-%d' % args) weights_path = densenet.train(extracted_path, output_path, blur=True, flip=True) weights_path_list.append(weights_path) archive_name = 'classifier.cameratrap.zip' archive_path = join(ibs.get_cachedir(), 'training', archive_name) ensemble_weights_path_list = [] for index, weights_path in enumerate(sorted(weights_path_list)): assert exists(weights_path) ensemble_weights_path = 'classifier.cameratrap.%d.weights' % (index, ) ut.copy(weights_path, ensemble_weights_path) ensemble_weights_path_list.append(ensemble_weights_path) ut.archive_files(archive_path, ensemble_weights_path_list, overwrite=True, common_prefix=True) return archive_path
def tozip(): re_fpath = ut.named_field('fpath', 'figure.*?[jp][pn]g') + '}' patterns = [ 'chapter4-application.tex', 'figdef4*', 'main.tex', 'def.tex', 'Crall*', 'thesis.cls', 'header*', 'colordef.tex', '*.bib' ] exclude_dirs = ['guts'] fpaths = sorted( ut.glob('.', patterns, recursive=True, exclude_dirs=exclude_dirs)) tup = ut.grep(re_fpath, fpath_list=fpaths, verbose=True) found_fpath_list, found_lines_list, found_lxs_list = tup fig_fpath_list = [] for line in ut.flatten(found_lines_list): if not line.startswith('%'): for match in re.finditer(re_fpath, line): fig_fpath = match.groupdict()['fpath'] if 'junc' not in fig_fpath and 'markov' not in fig_fpath and 'bayes' not in fig_fpath: fig_fpath_list += [fig_fpath] fpath_list = fig_fpath_list + fpaths ut.archive_files('chap4.zip', fpath_list)
def build_linux_zip_binaries(): fpath_list = ut.ls('dist/ibeis') archive_fpath = 'dist/ibeis-linux-binary.zip' ut.archive_files(archive_fpath, fpath_list) return archive_fpath
def classifier_multiclass_densenet_train(ibs, gid_list, label_list, ensembles=3, **kwargs): """ >>> import uuid >>> manifest_filepath = join(ibs.dbdir, 'flukebook_groundtruth.csv') >>> with open(manifest_filepath, 'r') as manifest_file: >>> line_list = manifest_file.readlines() >>> >>> label_dict = { >>> 'Left Dorsal Fin' : 'left_dorsal_fin', >>> 'Right Dorsal Fin' : 'right_dorsal_fin', >>> 'Tail Fluke' : 'tail_fluke', >>> } >>> >>> uuid_list = [] >>> label_list = [] >>> for line in line_list: >>> line = line.strip().split(',') >>> assert len(line) == 2 >>> uuid_, label_ = line >>> uuid_ = uuid.UUID(uuid_) >>> label_ = label_.strip() >>> print(uuid_, label_) >>> uuid_list.append(uuid_) >>> label_ = label_dict.get(label_, None) >>> assert label_ is not None >>> label_list.append(label_) >>> >>> gid_list = ibs.get_image_gids_from_uuid(uuid_list) >>> assert None not in gid_list >>> # archive_path = ibs.classifier_multiclass_densenet_train(gid_list, label_list) >>> ibs.classifier2_precision_recall_algo_display(test_gid_list=gid_list, test_label_list=label_list) """ from ibeis_cnn.ingest_ibeis import get_cnn_classifier_multiclass_training_images_pytorch from ibeis.algo.detect import densenet data_path = join(ibs.get_cachedir(), 'extracted-classifier-multiclass') extracted_path = get_cnn_classifier_multiclass_training_images_pytorch( ibs, gid_list, label_list, dest_path=data_path, image_size=densenet.INPUT_SIZE, **kwargs) weights_path_list = [] for ensemble_num in range(ensembles): args = (ensemble_num, ) output_path = join(ibs.get_cachedir(), 'training', 'classifier-multiclass-ensemble-%d' % args) weights_path = densenet.train(extracted_path, output_path, blur=True, flip=False) weights_path_list.append(weights_path) archive_name = 'classifier.multiclass.zip' archive_path = join(ibs.get_cachedir(), 'training', archive_name) ensemble_weights_path_list = [] for index, weights_path in enumerate(sorted(weights_path_list)): assert exists(weights_path) ensemble_weights_path = 'classifier.multiclass.%d.weights' % (index, ) ut.copy(weights_path, ensemble_weights_path) ensemble_weights_path_list.append(ensemble_weights_path) ut.archive_files(archive_path, ensemble_weights_path_list, overwrite=True, common_prefix=True) return archive_path