Ejemplo n.º 1
0
def test(gpath_list, canonical_weight_filepath=None, **kwargs):
    from detecttools.directory import Directory

    # Get correct weight if specified with shorthand
    archive_url = None

    ensemble_index = None
    if canonical_weight_filepath is not None and ':' in canonical_weight_filepath:
        assert canonical_weight_filepath.count(':') == 1
        canonical_weight_filepath, ensemble_index = canonical_weight_filepath.split(':')
        ensemble_index = int(ensemble_index)

    if canonical_weight_filepath in ARCHIVE_URL_DICT:
        archive_url = ARCHIVE_URL_DICT[canonical_weight_filepath]
        archive_path = ut.grab_file_url(archive_url, appname='ibeis', check_hash=True)
    else:
        raise RuntimeError('canonical_weight_filepath %r not recognized' % (canonical_weight_filepath, ))

    assert os.path.exists(archive_path)
    archive_path = ut.truepath(archive_path)

    ensemble_path = archive_path.strip('.zip')
    if not os.path.exists(ensemble_path):
        ut.unarchive_file(archive_path, output_dir=ensemble_path)

    assert os.path.exists(ensemble_path)
    direct = Directory(ensemble_path, include_file_extensions=['weights'], recursive=True)
    weights_path_list = direct.files()
    weights_path_list = sorted(weights_path_list)
    assert len(weights_path_list) > 0

    if ensemble_index is not None:
        assert 0 <= ensemble_index and ensemble_index < len(weights_path_list)
        weights_path_list = [ weights_path_list[ensemble_index] ]
        assert len(weights_path_list) > 0

    print('Using weights in the ensemble: %s ' % (ut.repr3(weights_path_list), ))
    result_list = test_ensemble(gpath_list, weights_path_list, **kwargs)
    for result in result_list:
        x0 = max(result['x0'], 0.0)
        y0 = max(result['y0'], 0.0)
        x1 = max(result['x1'], 0.0)
        y1 = max(result['y1'], 0.0)
        yield (x0, y0, x1, y1, )
Ejemplo n.º 2
0
    def __init__(pascald, dataset_path, **kwargs):
        com._kwargs(kwargs, 'object_min_width',        32)
        com._kwargs(kwargs, 'object_min_height',       32)
        com._kwargs(kwargs, 'mine_patches',            True)
        com._kwargs(kwargs, 'mine_negatives',          True)
        com._kwargs(kwargs, 'mine_width_min',          50)
        com._kwargs(kwargs, 'mine_width_max',          400)
        com._kwargs(kwargs, 'mine_height_min',         50)
        com._kwargs(kwargs, 'mine_height_max',         400)
        com._kwargs(kwargs, 'mine_max_attempts',       100)
        com._kwargs(kwargs, 'mine_max_keep',           10)
        com._kwargs(kwargs, 'mine_overlap_margin',     0.25)
        com._kwargs(kwargs, 'mine_exclude_categories', [])

        pascald.dataset_path = dataset_path
        pascald.absolute_dataset_path = os.path.realpath(dataset_path)

        direct = Directory(os.path.join(pascald.dataset_path, "Annotations") , include_file_extensions=["xml"])
        pascald.images = []
        files = direct.files()
        print("Loading Database")
        for i, filename in enumerate(files):
            if len(files) > 10:
                if i % (len(files) / 10) == 0:
                    print("%0.2f" % (float(i) / len(files)))
            pascald.images.append(PASCAL_Image(filename, pascald.absolute_dataset_path, **kwargs))
        print("    ...Loaded")

        pascald.categories_images = []
        pascald.categories_rois = []

        for image in pascald.images:
            temp = image.categories(unique=False, patches=True)
            pascald.categories_rois += temp
            pascald.categories_images += set(temp)

            if len(image.objects) == 0:
                pascald.categories_images += ["BACKGROUND"]

        pascald.distribution_images = com.histogram(pascald.categories_images)
        pascald.distribution_rois = com.histogram(pascald.categories_rois)
        pascald.rois = sum(pascald.distribution_rois.values())

        pascald.categories = sorted(set(pascald.categories_images))
Ejemplo n.º 3
0
def process_image_directory(project_name, size, reset=True):
    # Raw folders
    raw_path = abspath(join('..', 'data', 'raw'))
    processed_path = abspath(join('..', 'data', 'processed'))
    # Project folders
    project_raw_path = join(raw_path, project_name)
    project_processed_path = join(processed_path, project_name)

    # Load raw data
    direct = Directory(project_raw_path, include_extensions='images')

    # Reset / create paths if not exist
    if exists(project_processed_path) and reset:
        ut.remove_dirs(project_processed_path)
    ut.ensuredir(project_processed_path)

    # Process by resizing the images into the desired shape
    for file_path in direct.files():
        file_name = basename(file_path)
        print('Processing %r' % (file_name, ))
        image = cv2.imread(file_path)
        image = cv2.resize(image, size, interpolation=cv2.INTER_LANCZOS4)
        dest_path = join(project_processed_path, file_name)
        cv2.imwrite(dest_path, image)
Ejemplo n.º 4
0
def train_pyrf():
    # boosting = 3
    num_trees = 5
    category = 'zebra_grevys'

    #=================================
    # Train / Detect Configurations
    #=================================

    train_config = {
        'object_min_width':        32,
        'object_min_height':       32,
        'mine_negatives':          True,
        'mine_max_keep':           1,
        'mine_exclude_categories': [category],
        'mine_width_min':          128,
        'mine_width_max':          512,
        'mine_height_min':         128,
        'mine_height_max':         512,

        'neg_exclude_categories':  [category],
        'max_rois_pos':            900,
        'max_rois_neg':            550,
        'num_trees':               num_trees,
    }

    detect_config = {
        'save_detection_images':   True,
        'percentage_top':          0.40,
    }

    #=================================
    # Train / Detect Initialization
    #=================================

    # Create detector
    detector = Random_Forest_Detector()

    # Gather Dataset
    dataset_path = utool.unixpath('~/code/IBEIS2014/')
    dataset = IBEIS_Data(dataset_path, **train_config)

    results_path  = join(utool.unixpath('~/code/pyrf/results'), category)
    # pos_path      = join(results_path, 'train-positives')
    # neg_path      = join(results_path, 'train-negatives')
    # val_path      = join(results_path, 'val')
    test_path     = join(results_path, 'test')
    # test_pos_path = join(results_path, 'test-positives')
    # test_neg_path = join(results_path, 'test-negatives')
    detect_path   = join(results_path, 'detect')
    trees_path    = join(results_path, 'trees')

    # # Ensure result path for the category
    # # rmtreedir(results_path)
    # ensuredir(results_path)

    # for phase in range(1, boosting + 1):
    #     print("*********************")
    #     print("Phase: %s" % phase)
    #     print("*********************")
    #     # raw_input()
    #     # =================================
    #     # Train Random Forest
    #     #=================================
    #     detector.train(dataset, category, pos_path, neg_path, val_path,
    #                     test_path, test_pos_path, test_neg_path,
    #                     trees_path, reshuffle=(phase == 1), **train_config)

    #     if phase < boosting:
    #         #=================================
    #         # Detect using Random Forest
    #         #=================================

    #         # Load forest, so we don't have to reload every time
    #         forest = detector.load(trees_path, category + '-', num_trees=(phase * num_trees))
    #         detector.set_detect_params(**detect_config)

    #         # Ensure output detection paths
    #         rmtreedir(detect_path)
    #         ensuredir(detect_path)

    #         # Calculate error on test set
    #         direct = Directory(test_path, include_file_extensions=["jpg"])
    #         accuracy_list = []
    #         image_filepath_list = direct.files()
    #         dst_filepath_list   = [ join(detect_path, split(image_filepath)[1]) for image_filepath in image_filepath_list ]
    #         predictions_list = detector.detect_many(forest, image_filepath_list, dst_filepath_list, use_openmp=True)
    #         for index, (predictions, image_filepath) in enumerate(zip(predictions_list, image_filepath_list)):
    #             image_path, image_filename = split(image_filepath)
    #             image = dataset[image_filename]
    #             accuracy, true_pos, false_pos, false_neg = image.accuracy(predictions, category)
    #             accuracy_list.append(accuracy)
    #             progress = "%0.2f" % (float(index) / len(image_filepath_list))
    #             print("TEST %s %0.4f %s" % (image, accuracy, progress), end='\r')
    #             sys.stdout.flush()
    #             # image.show(prediction_list=predictions, category=category)
    #         print(' ' * 100, end='\r')
    #         print("TEST ERROR: %0.4f" % (1.0 - (float(sum(accuracy_list)) / len(accuracy_list))))

    #         #=================================
    #         # Eval and prep boosting train set
    #         #=================================
    #         detector.boosting(phase, forest, dataset, category, pos_path, neg_path,
    #                           test_pos_path, test_neg_path, detect_path)

    ####################################
    # New FAST
    ####################################

    detector = Random_Forest_Detector(
        scales='6 1.0 0.75 0.55 0.40 0.30 0.20'
    )

    # Ensure output detection paths
    detect_path_temp = detect_path + "_1"
    rmtreedir(detect_path_temp)
    ensuredir(detect_path_temp)

    # Load forest, so we don't have to reload every time
    forest = detector.load(trees_path, category + '-', num_trees=25)
    detector.set_detect_params(**detect_config)

    # Calculate error on test set
    direct = Directory(test_path, include_file_extensions=["jpg"])
    accuracy_list  = []
    true_pos_list  = []
    false_pos_list = []
    false_neg_list = []
    image_filepath_list = direct.files()
    dst_filepath_list   = [ join(detect_path_temp, split(image_filepath)[1]) for image_filepath in image_filepath_list ]
    predictions_list = detector.detect_many(forest, image_filepath_list, dst_filepath_list, use_openmp=True)
    for index, (predictions, image_filepath) in enumerate(zip(predictions_list, image_filepath_list)):
        image_path, image_filename = split(image_filepath)
        image = dataset[image_filename]
        accuracy, true_pos, false_pos, false_neg = image.accuracy(predictions, category)
        accuracy_list.append(accuracy)
        true_pos_list.append(true_pos)
        false_pos_list.append(false_pos)
        false_neg_list.append(false_neg)
        progress = "%0.2f" % (float(index) / len(image_filepath_list))
        print("TEST %s %0.4f %s" % (image, accuracy, progress), end='\r')
        sys.stdout.flush()
        # image.show(prediction_list=predictions, category=category)
    print(' ' * 100, end='\r')
    print("1 TEST ERROR     : %0.4f" % (1.0 - (float(sum(accuracy_list)) / len(accuracy_list))))
    print("1 TEST TRUE POS  : %d" % (sum(true_pos_list)))
    print("1 TEST FALSE POS : %d" % (sum(false_pos_list)))
    print("1 TEST FALSE NEG : %d" % (sum(false_neg_list)))

    ####################################
    # New SLOW
    ####################################

    detector = Random_Forest_Detector(
        scales='11 1.5 1.25 1.0 0.8 0.64 0.51 0.41 0.33 0.26 0.21 0.17'
    )

    # Ensure output detection paths
    detect_path_temp = detect_path + "_2"
    rmtreedir(detect_path_temp)
    ensuredir(detect_path_temp)

    # Load forest, so we don't have to reload every time
    forest = detector.load(trees_path, category + '-', num_trees=25)
    detector.set_detect_params(**detect_config)

    # Calculate error on test set
    direct = Directory(test_path, include_file_extensions=["jpg"])
    accuracy_list  = []
    true_pos_list  = []
    false_pos_list = []
    false_neg_list = []
    image_filepath_list = direct.files()
    dst_filepath_list   = [ join(detect_path_temp, split(image_filepath)[1]) for image_filepath in image_filepath_list ]
    predictions_list = detector.detect_many(forest, image_filepath_list, dst_filepath_list, use_openmp=True)
    for index, (predictions, image_filepath) in enumerate(zip(predictions_list, image_filepath_list)):
        image_path, image_filename = split(image_filepath)
        image = dataset[image_filename]
        accuracy, true_pos, false_pos, false_neg = image.accuracy(predictions, category)
        accuracy_list.append(accuracy)
        true_pos_list.append(true_pos)
        false_pos_list.append(false_pos)
        false_neg_list.append(false_neg)
        progress = "%0.2f" % (float(index) / len(image_filepath_list))
        print("TEST %s %0.4f %s" % (image, accuracy, progress), end='\r')
        sys.stdout.flush()
        # image.show(prediction_list=predictions, category=category)
    print(' ' * 100, end='\r')
    print("2 TEST ERROR     : %0.4f" % (1.0 - (float(sum(accuracy_list)) / len(accuracy_list))))
    print("2 TEST TRUE POS  : %d" % (sum(true_pos_list)))
    print("2 TEST FALSE POS : %d" % (sum(false_pos_list)))
    print("2 TEST FALSE NEG : %d" % (sum(false_neg_list)))
Ejemplo n.º 5
0
def numpy_processed_directory(project_name, numpy_ids_file_name='ids.npy',
                              numpy_x_file_name='X.npy',
                              numpy_y_file_name='y.npy', labels_file_name='labels.csv',
                              reset=True):
    # Raw folders
    processed_path = abspath(join('..', 'data', 'processed'))
    labels_path = abspath(join('..', 'data', 'labels'))
    numpy_path = abspath(join('..', 'data', 'numpy'))
    # Project folders
    project_processed_path = join(processed_path, project_name)
    project_labels_path = join(labels_path, project_name)
    project_numpy_path = join(numpy_path, project_name)
    # Project files
    project_numpy_ids_file_name = join(project_numpy_path, numpy_ids_file_name)
    project_numpy_x_file_name = join(project_numpy_path, numpy_x_file_name)
    project_numpy_y_file_name = join(project_numpy_path, numpy_y_file_name)
    project_numpy_labels_file_name = join(project_labels_path, labels_file_name)

    # Load raw data
    direct = Directory(project_processed_path, include_extensions='images')
    label_dict = {}
    for line in open(project_numpy_labels_file_name):
        line = line.strip().split(',')
        file_name = line[0].strip()
        label = line[1].strip()
        label_dict[file_name] = label

    # Reset / create paths if not exist
    if exists(project_numpy_path) and reset:
        ut.remove_dirs(project_numpy_path)
    ut.ensuredir(project_numpy_path)

    # Get shape for all images
    shape_x = list(cv2.imread(direct.files()[0]).shape)
    if len(shape_x) == 2:
        shape_x = shape_x + [1]
    shape_x = tuple([len(direct.files())] + shape_x[::-1])  # NOQA
    shape_y = shape_x[0:1]  # NOQA

    # Create numpy arrays
    # X = np.empty(shape_x, dtype=np.uint8)
    # y = np.empty(shape_y, dtype=np.uint8)
    ids = []
    X = []
    y = []

    # Process by loading images into the numpy array for saving
    for index, file_path in enumerate(direct.files()):
        file_name = basename(file_path)
        print('Processing %r' % (file_name, ))
        image = cv2.imread(file_path)
        try:
            label = label_dict[file_name]
            # X[index] = np.array(cv2.split(image))
            # y[index] = label
            # X.append(np.array(cv2.split(image)))  # Lasange format
            ids.append(file_name)
            X.append(image)  # cv2 format
            y.append(label)
        except KeyError:
            print('Cannot find label...skipping')
            # raw_input()

    ids = np.array(ids)
    X = np.array(X, dtype=np.uint8)
    # y = np.array(y, dtype=np.uint8)
    y = np.array(y)

    # Save numpy array
    print('  ids.shape = %r' % (ids.shape,))
    print('  ids.dtype = %r' % (ids.dtype,))
    print('  X.shape   = %r' % (X.shape,))
    print('  X.dtype   = %r' % (X.dtype,))
    print('  y.shape   = %r' % (y.shape,))
    print('  y.dtype   = %r' % (y.dtype,))
    np.save(project_numpy_ids_file_name, ids)
    np.save(project_numpy_x_file_name, X)
    np.save(project_numpy_y_file_name, y)
Ejemplo n.º 6
0
def numpy_processed_directory5(extracted_path, numpy_ids_file_name='ids.npy',
                               numpy_x_file_name='X.npy',
                               numpy_y_file_name='y.npy',
                               labels_file_name='labels.csv',
                               reset=True, verbose=False):
    print('Caching images into Numpy files with category vector...')

    raw_path = join(extracted_path, 'raw')
    labels_path = join(extracted_path, 'labels')

    # Project files
    project_numpy_ids_file_name = join(raw_path, numpy_ids_file_name)
    project_numpy_x_file_name = join(raw_path, numpy_x_file_name)
    project_numpy_y_file_name = join(labels_path, numpy_y_file_name)
    project_numpy_labels_file_name = join(labels_path, labels_file_name)

    # Load raw data
    direct = Directory(raw_path, include_extensions='images')
    label_dict = {}
    for line in open(project_numpy_labels_file_name):
        line = line.strip().split(',')
        file_name = line[0].strip()
        label = line[1].strip()
        label_list = label.split(';')
        label_list = [
            list(map(float, _.split('^')))
            for _ in label_list
        ]
        label = np.array(label_list)
        label_dict[file_name] = label

    # Create numpy arrays
    ids = []
    X = []
    y = []

    # Process by loading images into the numpy array for saving
    for index, file_path in enumerate(direct.files()):
        file_name = basename(file_path)
        if verbose:
            print('Processing %r' % (file_name, ))

        image = cv2.imread(file_path, -1)
        try:
            label = label_dict[file_name]
            ids.append(file_name)
            X.append(image)
            y.append(label)
        except KeyError:
            print('Cannot find label...skipping')

    ids = np.array(ids)
    X = np.array(X, dtype=np.uint8)
    y = np.array(y)

    # Save numpy array
    print('  ids.shape  = %r' % (ids.shape,))
    print('  ids.dtype  = %r' % (ids.dtype,))
    print('  X.shape    = %r' % (X.shape,))
    print('  X.dtype    = %r' % (X.dtype,))
    print('  y.shape    = %r' % (y.shape,))
    print('  y.dtype    = %r' % (y.dtype,))
    np.save(project_numpy_ids_file_name, ids)
    np.save(project_numpy_x_file_name, X)
    np.save(project_numpy_y_file_name, y)

    return project_numpy_ids_file_name, project_numpy_x_file_name, project_numpy_y_file_name
Ejemplo n.º 7
0
def numpy_processed_directory3(extracted_path, numpy_ids_file_name='ids.npy',
                               numpy_x_file_name='X.npy',
                               numpy_y_file_name='y.npy',
                               labels_file_name='labels.csv',
                               categories_file_name='categories.csv',
                               reset=True, verbose=False):
    print('Caching images into Numpy files with category vector...')

    raw_path = join(extracted_path, 'raw')
    labels_path = join(extracted_path, 'labels')

    # Project files
    project_numpy_ids_file_name = join(raw_path, numpy_ids_file_name)
    project_numpy_x_file_name = join(raw_path, numpy_x_file_name)
    project_numpy_y_file_name = join(labels_path, numpy_y_file_name)
    project_numpy_labels_file_name = join(labels_path, labels_file_name)
    project_numpy_categories_file_name = join(labels_path, categories_file_name)

    category_list = []
    for line in open(project_numpy_categories_file_name):
        category = line.strip()
        if len(category) > 0:
            category_list.append(category)

    # Load raw data
    direct = Directory(raw_path, include_extensions='images')
    label_dict = {}
    count_dict = {}
    for line in open(project_numpy_labels_file_name):
        line = line.strip().split(',')
        file_name = line[0].strip()
        label = line[1].strip()
        label_list = label.split(';')
        label_set = set(label_list)
        label = [ 1 if category_ in label_set else 0 for category_ in category_list]
        assert 1 in label
        count = label.count(1)
        if count not in count_dict:
            count_dict[count] = 0
        count_dict[count] += 1
        label_dict[file_name] = label

    print('count_dict = %s' % (ut.repr3(count_dict), ))

    # Get shape for all images
    shape_x = list(cv2.imread(direct.files()[0]).shape)
    if len(shape_x) == 2:
        shape_x = shape_x + [1]
    shape_x = tuple([len(direct.files())] + shape_x[::-1])  # NOQA
    shape_y = shape_x[0:1]  # NOQA

    # Create numpy arrays
    # X = np.empty(shape_x, dtype=np.uint8)
    # y = np.empty(shape_y, dtype=np.uint8)
    ids = []
    X = []
    y = []

    # Process by loading images into the numpy array for saving
    for index, file_path in enumerate(direct.files()):
        file_name = basename(file_path)
        if verbose:
            print('Processing %r' % (file_name, ))
        image = cv2.imread(file_path)
        try:
            label = np.array(label_dict[file_name])
            # X[index] = np.array(cv2.split(image))
            # y[index] = label
            # X.append(np.array(cv2.split(image)))  # Lasange format
            ids.append(file_name)
            X.append(image)  # cv2 format
            y.append(label)
        except KeyError:
            print('Cannot find label...skipping')
            # raw_input()

    ids = np.array(ids)
    X = np.array(X, dtype=np.uint8)
    # y = np.array(y, dtype=np.uint8)
    y = np.vstack(y)

    # Save numpy array
    print('  ids.shape  = %r' % (ids.shape,))
    print('  ids.dtype  = %r' % (ids.dtype,))
    print('  X.shape    = %r' % (X.shape,))
    print('  X.dtype    = %r' % (X.dtype,))
    print('  y.shape    = %r' % (y.shape,))
    print('  y.dtype    = %r' % (y.dtype,))
    print('  categories = %r' % (category_list,))
    np.save(project_numpy_ids_file_name, ids)
    np.save(project_numpy_x_file_name, X)
    np.save(project_numpy_y_file_name, y)

    return project_numpy_ids_file_name, project_numpy_x_file_name, project_numpy_y_file_name
Ejemplo n.º 8
0
    exportedmdb_fpath  = join(csv_fpath, 'Individual sightings.csv')

    utool.checkpath(activity_csv_fpath, verbose=True)
    utool.checkpath(exportedmdb_fpath, verbose=True)

    with open(join(activity_csv_fpath), 'r') as file_:
        lines = file_.read()
        for line in lines.splitlines()[1:]:
            line = [ item.strip() for item in line.strip().split(',')]
            _id = line[2]
            if _id not in activities:
                activities[_id] = [line[col] for col in columns]

    originals = join(prefix, 'Ol_pejeta_zebra_photos2__1GB')
    images = Directory(originals)
    image_set = set(images.files())
    print(images)
    exts = []
    for image in images.files():
        exts.append(image.split('.')[-1])
    exts = list(set(exts))
    print('EXTENSIONS: %r ' % (exts,))

    print('''
          =====================
          PROCESSING IMAGESETS
          =====================
          ''')
    used = []
    # e ncounters = open(join(prefix, 'e ncounters.csv'),'w')
    # animals = open(join(prefix, 'animals.csv'),'w')
Ejemplo n.º 9
0
def test(gpath_list,
         classifier_weight_filepath=None,
         return_dict=False,
         multiclass=False,
         **kwargs):
    from detecttools.directory import Directory
    # Get correct weight if specified with shorthand
    archive_url = None

    ensemble_index = None
    if classifier_weight_filepath is not None and ':' in classifier_weight_filepath:
        assert classifier_weight_filepath.count(':') == 1
        classifier_weight_filepath, ensemble_index = classifier_weight_filepath.split(
            ':')
        ensemble_index = int(ensemble_index)

    if classifier_weight_filepath in ARCHIVE_URL_DICT:
        archive_url = ARCHIVE_URL_DICT[classifier_weight_filepath]
        archive_path = ut.grab_file_url(archive_url,
                                        appname='ibeis',
                                        check_hash=True)
    else:
        print('classifier_weight_filepath %r not recognized' %
              (classifier_weight_filepath, ))
        raise RuntimeError

    assert os.path.exists(archive_path)
    archive_path = ut.truepath(archive_path)

    ensemble_path = archive_path.strip('.zip')
    if not os.path.exists(ensemble_path):
        ut.unarchive_file(archive_path, output_dir=ensemble_path)

    assert os.path.exists(ensemble_path)
    direct = Directory(ensemble_path,
                       include_file_extensions=['weights'],
                       recursive=True)
    weights_path_list = direct.files()
    weights_path_list = sorted(weights_path_list)
    assert len(weights_path_list) > 0

    kwargs.pop('classifier_algo', None)

    print('Using weights in the ensemble, index %r: %s ' % (
        ensemble_index,
        ut.repr3(weights_path_list),
    ))
    result_list = test_ensemble(gpath_list,
                                weights_path_list,
                                classifier_weight_filepath,
                                ensemble_index,
                                multiclass=multiclass,
                                **kwargs)
    for result in result_list:
        best_key = None
        best_score = -1.0
        for key, score in result.items():
            if score > best_score:
                best_key = key
                best_score = score
        assert best_score >= 0.0 and best_key is not None
        if return_dict:
            yield best_score, best_key, result
        else:
            yield best_score, best_key
Ejemplo n.º 10
0
    def train(rf, train_pos_cpath_list, train_neg_cpath_list, trees_path, **kwargs):
        """
            Train a new forest with the given positive chips and negative chips.

            Args:
                train_pos_chip_path_list (list of str): list of positive training chips
                train_neg_chip_path_list (list of str): list of negative training chips
                trees_path (str): string path of where the newly trained trees are to be saved

            Kwargs:
                chips_norm_width (int, optional): Chip normalization width for resizing;
                    the chip is resized to have a width of chips_norm_width and
                    whatever resulting height in order to best match the original
                    aspect ratio; defaults to 128

                    If both chips_norm_width and chips_norm_height are specified,
                    the original aspect ratio of the chip is not respected
                chips_norm_height (int, optional): Chip normalization height for resizing;
                    the chip is resized to have a height of chips_norm_height and
                    whatever resulting width in order to best match the original
                    aspect ratio; defaults to None

                    If both chips_norm_width and chips_norm_height are specified,
                    the original aspect ratio of the chip is not respected
                chips_prob_flip_horizontally (float, optional): The probability
                    that a chips is flipped horizontally before training to make
                    the training set invariant to horizontal flips in the image;
                    defaults to 0.5; 0.0 <= chips_prob_flip_horizontally <= 1.0
                chips_prob_flip_vertically (float, optional): The probability
                    that a chips is flipped vertivcally before training to make
                    the training set invariant to vertical flips in the image;
                    defaults to 0.5; 0.0 <= chips_prob_flip_vertically <= 1.0
                patch_width (int, optional): the width of the patches for extraction
                    in the tree; defaults to 32; patch_width > 0
                patch_height (int, optional): the height of the patches for extraction
                    in the tree; defaults to 32; patch_height > 0
                patch_density (float, optional): the number of patches to extract from
                    each chip as a function of density; the density is calculated as:
                        samples = patch_density * [(chip_width * chip_height) / (patch_width * patch_height)]
                    and specifies how many times a particular pixel is sampled
                    from the chip; defaults to 4.0; patch_density > 0
                trees_num (int, optional): the number of trees to train in parallel;
                    defaults to 10
                trees_offset (int, optional): the tree number that begins the sequence
                    of when a tree is trained; defaults to None

                    If None is specified, the trees_offset value is automatically guessed
                    by using the number of files in trees_path

                    Tree model files are overwritten if the offset has overlap with
                    previouly generated trees
                trees_max_depth (int, optional): the maximum depth of the tree during
                    training, this can used for regularization; defaults to 16
                trees_max_patches (int, optional): the maximum number of patches that
                    should be extracted for training between positives AND negatives
                    (the detector attempts to balance between the number of positive
                    and negative patches to be roughly the same in quantity);
                    defaults to 64000
                trees_leaf_size (int, optional): the number of patches in a node that
                    specifies the threshold for becoming a leaf; defaults to 20

                    A node becomes a leaf under two conditions:
                        1.) The maximum tree depth has been reached (trees_max_depth)
                        2.) The patches in the node is less than trees_leaf_size and
                            is stopped prematurely
                trees_pixel_tests (int, optional): the number of pixel tests to perform
                    at each node; defaults to 2000
                trees_prob_optimize_mode (float, optional): The probability of the
                    tree optimizing between classification and regression; defaults to
                    0.5
                serial (bool, optional): flag to signify if to run training in serial;
                    defaults to False
                verbose (bool, optional): verbose flag; defaults to object's verbose or
                    selectively enabled for this function

            Returns:
                None
        """
        # Default values
        params = odict([
            ('chips_norm_width',             128),
            ('chips_norm_height',            None),
            ('chips_prob_flip_horizontally', 0.5),
            ('chips_prob_flip_vertically',   0.0),
            ('patch_width',                  32),
            ('patch_height',                 32),
            ('patch_density',                4.0),
            ('trees_num',                    10),
            ('trees_offset',                 None),
            ('trees_max_depth',              16),
            ('trees_max_patches',            64000),
            ('trees_leaf_size',              20),
            ('trees_pixel_tests',            10000),
            ('trees_prob_optimize_mode',     0.5),
            ('serial',                       False),
            ('verbose',                      rf.verbose),
            ('quiet',                        rf.quiet),
        ])
        #params.update(kwargs)
        ut.update_existing(params, kwargs)
        # Make the tree path absolute
        trees_path = abspath(trees_path)

        # cout << "AIM FOR A SPLIT OF 24k - 32k POSITIVE & NEGATIVE PATCHES EACH FOR GOOD REGULARIZATION AT DEPTH 16" << endl;

        # Ensure the trees_path exists
        ut.ensuredir(trees_path)
        data_path = join(trees_path, 'data')
        if isdir(data_path):
            shutil.rmtree(data_path)
        ut.ensuredir(data_path)
        data_path_pos = join(data_path, 'pos')
        ut.ensuredir(data_path_pos)
        data_path_neg = join(data_path, 'neg')
        ut.ensuredir(data_path_neg)

        # Try to figure out the correct tree offset
        if params['trees_offset'] is None:
            direct = Directory(trees_path, include_file_extensions=['txt'])
            params['trees_offset'] = len(direct.files()) + 1
            if not params['quiet']:
                print('[pyrf py] Auto Tree Offset: %d' % params['trees_offset'])

        # Data integrity
        assert params['chips_norm_width'] is None or params['chips_norm_width'] >= params['patch_width'], \
            'Normalization width too small for patch width'
        assert params['chips_norm_height'] is None or params['chips_norm_height'] >= params['patch_height'], \
            'Normalization height too small for patch height'
        assert params['patch_width'] > 0, \
            'Patch width must be positive'
        assert params['patch_height'] > 0, \
            'Patch height must be positive'
        assert params['patch_density'] > 0.0, \
            'Patch density must be positive'
        assert 0.0 <= params['chips_prob_flip_horizontally'] and params['chips_prob_flip_horizontally'] <= 1.0, \
            'Horizontal flip probability must be between 0 and 1'
        assert 0.0 <= params['chips_prob_flip_vertically'] and params['chips_prob_flip_vertically'] <= 1.0, \
            'Vertical flip probability must be between 0 and 1'
        assert params['trees_num'] > 0, \
            'Number of trees must be positive'
        assert params['trees_offset'] >= 0, \
            'Number of trees must be non-negative'
        assert params['trees_max_depth'] > 1, \
            'Tree depth must be greater than 1'
        assert params['trees_max_patches'] % 2 == 0 and params['trees_max_patches'] > 0, \
            'A tree must have an even (positive) number of patches'
        assert 0.0 <= params['trees_prob_optimize_mode'] and params['trees_prob_optimize_mode'] <= 1.0, \
            'Tree optimization mode probability must be between 0 and 1 (inclusive)'
        assert all( [ exists(train_pos_cpath) for train_pos_cpath in train_pos_cpath_list ] ), \
            'At least one specified positive chip path does not exist'
        assert all( [ exists(train_neg_cpath) for train_neg_cpath in train_neg_cpath_list ] ), \
            'At least one specified positive chip path does not exist'
        # We will let the C++ code perform the patch size checks

        if not params['quiet']:
            print('[pyrf py] Caching positives into %r' % (data_path_pos, ))
        train_pos_chip_filename_list = _cache_data(train_pos_cpath_list, data_path_pos, **params)

        if not params['quiet']:
            print('[pyrf py] Caching negatives into %r' % (data_path_neg, ))
        train_neg_chip_filename_list = _cache_data(train_neg_cpath_list, data_path_neg, **params)

        # We no longer need these parameters (and they should not be transferred to the C++ library)
        del params['chips_norm_width']
        del params['chips_norm_height']
        del params['chips_prob_flip_horizontally']
        del params['chips_prob_flip_vertically']

        # Run training algorithm
        params_list = [
            data_path_pos,
            _cast_list_to_c(ensure_bytes_strings(train_pos_chip_filename_list), C_CHAR),
            len(train_pos_chip_filename_list),
            data_path_neg,
            _cast_list_to_c(ensure_bytes_strings(train_neg_chip_filename_list), C_CHAR),
            len(train_neg_chip_filename_list),
            trees_path,
        ] + list(params.values())
        RF_CLIB.train(rf.detector_c_obj, *params_list)
        if not params['quiet']:
            print('\n\n[pyrf py] *************************************')
            print('[pyrf py] Training Completed')
Ejemplo n.º 11
0
 def train_folder(rf, train_pos_path, train_neg_path, trees_path, **kwargs):
     direct = Directory(train_pos_path, include_file_extensions='images')
     train_pos_cpath_list = direct.files()
     direct = Directory(train_neg_path, include_file_extensions='images')
     train_neg_cpath_list = direct.files()
     return rf.train(train_pos_cpath_list, train_neg_cpath_list, trees_path, **kwargs)
Ejemplo n.º 12
0
    exportedmdb_fpath  = join(csv_fpath, 'Individual sightings.csv')

    utool.checkpath(activity_csv_fpath, verbose=True)
    utool.checkpath(exportedmdb_fpath, verbose=True)

    with open(join(activity_csv_fpath), 'r') as file_:
        lines = file_.read()
        for line in lines.splitlines()[1:]:
            line = [ item.strip() for item in line.strip().split(',')]
            _id = line[2]
            if _id not in activities:
                activities[_id] = [line[col] for col in columns]

    originals = join(prefix, 'Ol_pejeta_zebra_photos2__1GB')
    images = Directory(originals)
    image_set = set(images.files())
    print(images)
    exts = []
    for image in images.files():
        exts.append(image.split('.')[-1])
    exts = list(set(exts))
    print('EXTENSIONS: %r ' % (exts,))

    print('''
          =====================
          PROCESSING ENCOUNTERS
          =====================
          ''')
    used = []
    # encounters = open(join(prefix, 'encounters.csv'),'w')
    # animals = open(join(prefix, 'animals.csv'),'w')
Ejemplo n.º 13
0
def status():
    cars = {}
    # STEP 1 - GPS
    try:
        gps_path = join(DEFAULT_DATA_DIR, 'gps')
        direct = Directory(gps_path,
                           recursive=True,
                           include_file_extensions=[])
        for direct_car in direct.directory_list:
            base_car = direct_car.base()
            car_path = join(gps_path, base_car)
            # Establish flags
            flags = {}
            flags['submitted_gpx'] = exists(join(car_path, 'track.gpx'))
            flags['submitted_generated_json'] = exists(
                join(car_path, 'track.json'))
            cars[base_car] = {'gps': flags}
            cars[base_car]['persons'] = {}
    except:
        print("No GPS data submitted yet")

    # STEP 2 - IMAGES
    try:
        images_path = join(DEFAULT_DATA_DIR, 'images')
        direct = Directory(images_path,
                           recursive=True,
                           include_file_extensions=[])
        for direct_car in direct.directory_list:
            base_car = direct_car.base()
            if base_car not in cars:
                cars[base_car] = {}
            cars[base_car]['persons'] = {}
            # Construct dict for person
            for direct_person in direct_car.directory_list:
                base_person = direct_person.base()
                person_path = join(images_path, base_car, base_person)
                # Establish flags, inherit GPS for person if car has GPS flags
                if 'gps' in cars[base_car]:
                    flags = dict(cars[base_car]['gps'])
                else:
                    flags = {}
                flags['submitted_images'] = True
                flags['submitted_first'] = exists(
                    join(person_path, 'first.jpg'))
                flags['submitted_last'] = exists(join(person_path, 'last.jpg'))
                # Giraffe flags
                giraffe_path = join(person_path, 'giraffe')
                if isdir(giraffe_path):
                    flags['submitted_giraffe'] = True
                    temp = Directory(giraffe_path,
                                     include_file_extensions='images')
                    flags['submitted_giraffe_number'] = len(temp.files())
                # Zebra flags
                zebra_path = join(person_path, 'zebra')
                if isdir(zebra_path):
                    flags['submitted_zebra'] = True
                    temp = Directory(zebra_path,
                                     include_file_extensions='images')
                    flags['submitted_zebra_number'] = len(temp.files())
                # Generated offset
                offset_path = join(person_path, 'offset.json')
                if exists(offset_path):
                    flags['submitted_generated_offset'] = True
                    with open(offset_path, 'r') as off:
                        data = json.load(off)
                        offset = data.get('offset', 0.0)
                    flags['submitted_generated_offset_number'] = offset
                # Assign flags to person
                cars[base_car]['persons'][base_person] = flags
            cars[base_car]['persons_number'] = len(
                cars[base_car]['persons'].keys())
    except:
        print("No Image data submitted yet")

    # STEP 3 - ANALYSIS
    try:
        analysis_path = join(DEFAULT_DATA_DIR, 'analysis')
        direct = Directory(analysis_path,
                           recursive=True,
                           include_file_extensions=[])
        for direct_car in direct.directory_list:
            base_car = direct_car.base()
            if base_car not in cars:
                cars[base_car] = {}
            # Append to dict for person
            for direct_person in direct_car.directory_list:
                base_person = direct_person.base()
                person_path = join(analysis_path, base_car, base_person)
                # Establish flags
                if base_person in cars[base_car]['persons']:
                    flags = dict(cars[base_car]['persons'][base_person])
                else:
                    flags = {}
                flags['analyzed'] = True
                flags['analyzed_review'] = exists(
                    join(person_path, 'review.flag'))
                # Giraffe flags
                giraffe_path = join(person_path, 'giraffe')
                if isdir(giraffe_path):
                    flags['analyzed_giraffe'] = True
                    flags['analyzed_generated_giraffe_confidences'] = exists(
                        join(giraffe_path, 'confidences.json'))
                    temp = Directory(giraffe_path,
                                     include_file_extensions=['jpg', 'json'])
                    flags['analyzed_giraffe_number'] = (
                        len(temp.files()) - 1
                    ) / 4.0  # 4 images per detection generated (minus 1 confidences)
                    if not flags[
                            'analyzed_generated_giraffe_confidences'] and flags[
                                'analyzed_giraffe_number'] == -0.25:
                        flags['analyzed_giraffe_number'] = 0.0

                # Zebra flags
                zebra_path = join(person_path, 'zebra')
                if isdir(zebra_path):
                    flags['analyzed_zebra'] = True
                    flags['analyzed_generated_zebra_confidences'] = exists(
                        join(zebra_path, 'confidences.json'))
                    temp = Directory(zebra_path,
                                     include_file_extensions=['jpg', 'json'])
                    flags['analyzed_zebra_number'] = (
                        len(temp.files()) - 1
                    ) / 4.0  # 4 images per detection generated (minus 1 confidences)
                    if not flags[
                            'analyzed_generated_zebra_confidences'] and flags[
                                'analyzed_zebra_number'] == -0.25:
                        flags['analyzed_zebra_number'] = 0.0
                # Assign flags to person
                cars[base_car]['persons'][base_person] = flags
            cars[base_car]['persons_number'] = len(
                cars[base_car]['persons'].keys())
    except:
        print("No images analyzed yet")

    # STEP 4 - PDFS
    try:
        pdfs_path = join(DEFAULT_DATA_DIR, 'pdfs')
        direct = Directory(pdfs_path,
                           recursive=True,
                           include_file_extensions=[])
        for direct_car in direct.directory_list:
            base_car = direct_car.base()
            if base_car not in cars:
                cars[base_car] = {}
            # Append to dict for person
            for direct_person in direct_car.directory_list:
                base_person = direct_person.base()
                person_path = join(pdfs_path, base_car, base_person)
                # Establish flags
                if base_person in cars[base_car]['persons']:
                    flags = dict(cars[base_car]['persons'][base_person])
                else:
                    flags = {}
                flags['reviewed'] = True
                flags['reviewed_posted'] = exists(
                    join(person_path, 'content.html'))
                flags['reviewed_rendered'] = exists(
                    join(person_path, 'content.pdf'))
                flags['reviewed_printed'] = exists(
                    join(person_path, 'printed.flag'))
                # Assign flags to person
                cars[base_car]['persons'][base_person] = flags
            cars[base_car]['persons_number'] = len(
                cars[base_car]['persons'].keys())
    except:
        print("No PDFs rendered yet")

    extra = {}
    extra['cars'] = cars
    extra['cars_number'] = len(cars.keys())
    return sf.response(**extra)
Ejemplo n.º 14
0
def convert_ggr2018_to_ibeis(ggr_path, dbdir=None, purge=True, dry_run=False,
                             apply_updates=True, **kwargs):
    r"""Convert the raw GGR2 (2018) data to an ibeis database.

    Args
        ggr_path (str): Directory to folder *containing* raw GGR 2018 data
        dbdir (str): Output directory

    CommandLine:
        python -m ibeis convert_ggr2018_to_ibeis

    Example:
        >>> # SCRIPT
        >>> from ibeis.dbio.ingest_ggr import *  # NOQA
        >>> default_ggr_path = join('/', 'data', 'ibeis', 'GGR2', 'GGR2018data')
        >>> default_dbdir = join('/', 'data', 'ibeis', 'GGR2-IBEIS')
        >>> dbdir = ut.get_argval('--dbdir', type_=str, default=default_dbdir)
        >>> ggr_path = ut.get_argval('--ggr', type_=str, default=default_ggr_path)
        >>> result = convert_ggr2018_to_ibeis(ggr_path, dbdir=dbdir, purge=False, dry_run=True, apply_updates=False)
        >>> print(result)
    """
    ALLOWED_NUMBERS = list(range(1, 250))
    ALLOWED_LETTERS = ['A', 'B', 'C', 'D', 'E', 'F']

    ################################################################################

    if apply_updates:
        _fix_ggr2018_directory_structure(ggr_path)

    ################################################################################

    blacklist_filepath_set = set([
        join(ggr_path, 'Cameras info.numbers'),
        join(ggr_path, 'Cameras info.xlsx'),
        join(ggr_path, 'GGR_photos_MRC_29.1.18.ods'),
        join(ggr_path, 'Cameras info-2.numbers'),
    ])

    # Check root files
    direct = Directory(ggr_path)
    for filepath in direct.files(recursive=False):
        try:
            assert filepath in blacklist_filepath_set
            ut.delete(filepath)
        except AssertionError:
            print('Unresolved root file found in %r' % (filepath, ))
            continue

    ################################################################################

    if purge:
        ut.delete(dbdir)
    ibs = ibeis.opendb(dbdir=dbdir)

    ################################################################################

    # Check folder structure
    assert exists(ggr_path)
    direct = Directory(ggr_path, recursive=0)
    direct1_list = direct.directories()
    direct1_list.sort(key=lambda x: int(x.base()), reverse=False)
    for direct1 in direct1_list:
        if not dry_run:
            print('Processing directory: %r' % (direct1, ))
        base1 = direct1.base()

        try:
            int(base1)
        except ValueError:
            print('Error found in %r' % (direct1, ))
            continue

        try:
            assert len(direct1.files(recursive=False)) == 0
        except AssertionError:
            print('Files found in %r' % (direct1, ))
            continue

        seen_letter_list = []
        direct1_ = Directory(direct1.absolute_directory_path, recursive=0)
        direct2_list = direct1_.directories()
        direct2_list.sort(key=lambda x: x.base(), reverse=False)
        for direct2 in direct2_list:
            base2 = direct2.base()

            try:
                assert base2.startswith(base1)
            except AssertionError:
                print('Folder name heredity conflict %r with %r' % (direct2, direct1, ))
                continue

            try:
                assert len(base2) >= 2
                assert ' ' not in base2
                number = base2[:-1]
                letter = base2[-1]
                number = int(number)
                letter = letter.upper()
                assert number in ALLOWED_NUMBERS
                assert letter in ALLOWED_LETTERS
                seen_letter_list.append(letter)
            except ValueError:
                print('Error found in %r' % (direct2, ))
                continue
            except AssertionError:
                print('Folder name format error found in %r' % (direct2, ))
                continue

            direct2_ = Directory(direct2.absolute_directory_path, recursive=True, images=True)
            try:
                assert len(direct2_.directories()) == 0
            except AssertionError:
                print('Folders exist in file only level %r' % (direct2, ))
                continue

            filepath_list = sorted(direct2_.files())

            if not dry_run:
                try:
                    gid_list = ibs.add_images(filepath_list)
                    gid_list = ut.filter_Nones(gid_list)
                    gid_list = sorted(list(set(gid_list)))

                    imageset_text = 'GGR2,%d,%s' % (number, letter, )
                    note_list = [
                        '%s,%05d' % (imageset_text, index + 1)
                        for index, gid in enumerate(gid_list)
                    ]
                    ibs.set_image_notes(gid_list, note_list)
                    ibs.set_image_imagesettext(gid_list, [imageset_text] * len(gid_list))
                except Exception as ex:  # NOQA
                    ut.embed()

        seen_letter_set = set(seen_letter_list)
        try:
            assert len(seen_letter_set) == len(seen_letter_list)
        except AssertionError:
            print('Duplicate letters in %r with letters %r' % (direct1, seen_letter_list, ))
            continue

        try:
            assert 'A' in seen_letter_set
        except AssertionError:
            print('WARNING: A camera not found in %r' % (direct1, ))
            continue

    return ibs
Ejemplo n.º 15
0
def _fix_ggr2018_directory_structure(ggr_path):

    # Manual fixes for bad directories

    src_uri = join(ggr_path, 'Clarine\ Plane\ Kurungu/')
    dst_uri = join(ggr_path, '231/')
    ut.ensuredir(dst_uri)
    dst_uri = join(dst_uri, '231B/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, 'Alex\ Peltier\ -\ Plane\ -\ Ngurnit/giraffe\ grevy\ count\ feb\ 18/')
    dst_uri = join(ggr_path, '232/')
    ut.ensuredir(dst_uri)
    dst_uri = join(dst_uri, '232B/')
    ut.rsync(src_uri, dst_uri)
    src_uri = src_uri.replace('\\', '')
    src_uri = '/'.join(src_uri.split('/')[:-2])
    ut.delete(src_uri)

    src_uri = join(ggr_path, 'Mint\ Media\ Footage', 'Mpala\ day\ 1\ spark', 'PANORAMA/')
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, 'Mint\ Media\ Footage', 'Mpala\ day\ 1/')
    dst_uri = join(ggr_path, '233/')
    ut.ensuredir(dst_uri)
    dst_uri = join(dst_uri, '233B/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, 'Mint\ Media\ Footage', 'Mpala\ day\ 1\ spark/')
    dst_uri = join(ggr_path, '233', '233B/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, 'Mint\ Media\ Footage', 'Mpala\ day2\ /')
    dst_uri = join(ggr_path, '233', '233B/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, 'Mint\ Media\ Footage', 'Mpala\ day\ 2\ spark/')
    dst_uri = join(ggr_path, '233', '233B/')
    ut.rsync(src_uri, dst_uri)
    src_uri = src_uri.replace('\\', '')
    src_uri = '/'.join(src_uri.split('/')[:-2])
    ut.delete(src_uri)

    src_uri = join(ggr_path, '103\ \(1\)/')
    dst_uri = join(ggr_path, '103/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '103\ \(ccef473b\)/')
    dst_uri = join(ggr_path, '103/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '108\ \(1\)/')
    dst_uri = join(ggr_path, '108/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '226A\ \(Shaba\ Funan\ Camp\)/')
    dst_uri = join(ggr_path, '226/')
    ut.ensuredir(dst_uri)
    dst_uri = join(dst_uri, '226A/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '121/*.*')
    dst_uri = join(ggr_path, '121', '121A/')
    ut.rsync(src_uri, dst_uri)
    for src_filepath in ut.glob(src_uri.replace('\\', '')):
        ut.delete(src_filepath)

    src_uri = join(ggr_path, '54', '54A\(16\)/')
    dst_uri = join(ggr_path, '54', '54A/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '54', '54B\(16\)/')
    dst_uri = join(ggr_path, '54', '54B/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '87', '87/')
    dst_uri = join(ggr_path, '87', '87A/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '223', 'A/')
    dst_uri = join(ggr_path, '223', '223A/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '223', 'B/')
    dst_uri = join(ggr_path, '223', '223B/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '14', '15A/')
    dst_uri = join(ggr_path, '14', '14A/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '73/')
    dst_uri = join(ggr_path, '85/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '117', '115A/')
    dst_uri = join(ggr_path, '117', '117A/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '200', '200\ A/')
    dst_uri = join(ggr_path, '200', '200A/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '200', '200\ B/')
    dst_uri = join(ggr_path, '200', '200B/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '200', '200\ F/')
    dst_uri = join(ggr_path, '200', '200F/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '200', '200A/')
    dst_uri = join(ggr_path, '201/')
    ut.ensuredir(dst_uri)
    dst_uri = join(dst_uri, '201A/')
    ut.rsync(src_uri, dst_uri)
    # ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '200', '201\ E/')
    dst_uri = join(ggr_path, '201', '201E/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '200', '201\ F/')
    dst_uri = join(ggr_path, '201', '201F/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '200', '200A/')
    dst_uri = join(ggr_path, '202/')
    ut.ensuredir(dst_uri)
    dst_uri = join(dst_uri, '202A/')
    ut.rsync(src_uri, dst_uri)
    # ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '200', '202\ B/')
    dst_uri = join(ggr_path, '202', '202B/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '200', '202\ F/')
    dst_uri = join(ggr_path, '202', '202F/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '230', '230A', 'El\ Karama/*.*')
    dst_uri = join(ggr_path, '230', '230A/')
    ut.rsync(src_uri, dst_uri)
    src_uri = src_uri.replace('\\', '')
    src_uri = '/'.join(src_uri.split('/')[:-1])
    ut.delete(src_uri)

    src_uri = join(ggr_path, '136', '136B', '136B\ Grevys\ Rally/*.*')
    dst_uri = join(ggr_path, '136', '136B/')
    ut.rsync(src_uri, dst_uri)
    src_uri = src_uri.replace('\\', '')
    src_uri = '/'.join(src_uri.split('/')[:-1])
    ut.delete(src_uri)

    src_uri = join(ggr_path, '160', '160E', '104DUSIT')
    if exists(src_uri):
        direct = Directory(src_uri, recursive=False)
        filename_list = direct.files()
        for filename in sorted(filename_list):
            dst_uri = filename.replace('104DUSIT/', '').replace('.JPG', '_.JPG')
            assert not exists(dst_uri)
            ut.rsync(filename, dst_uri)
        ut.delete(src_uri)

    src_uri = join(ggr_path, '222', '222B', '102DUSIT')
    if exists(src_uri):
        direct = Directory(src_uri, recursive=False)
        filename_list = direct.files()
        for filename in sorted(filename_list):
            dst_uri = filename.replace('102DUSIT/', '').replace('.JPG', '_.JPG')
            assert not exists(dst_uri)
            ut.rsync(filename, dst_uri)
        ut.delete(src_uri)

    # Errors found by QR codes

    # No conflicts
    src_uri = join(ggr_path, '5', '5A/')
    dst_uri = join(ggr_path, '5', '5B/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '14', '14A/')
    dst_uri = join(ggr_path, '14', '14B/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '118', '118A/')
    dst_uri = join(ggr_path, '192')
    ut.ensuredir(dst_uri)
    dst_uri = join(dst_uri, '192A/')
    ut.rsync(src_uri, dst_uri)
    src_uri = src_uri.replace('\\', '')
    src_uri = '/'.join(src_uri.split('/')[:-2])
    ut.delete(src_uri)

    src_uri = join(ggr_path, '119', '119A/')
    dst_uri = join(ggr_path, '189')
    ut.ensuredir(dst_uri)
    dst_uri = join(dst_uri, '189A/')
    ut.rsync(src_uri, dst_uri)
    src_uri = src_uri.replace('\\', '')
    src_uri = '/'.join(src_uri.split('/')[:-2])
    ut.delete(src_uri)

    src_uri = join(ggr_path, '120', '120A/')
    dst_uri = join(ggr_path, '190')
    ut.ensuredir(dst_uri)
    dst_uri = join(dst_uri, '190A/')
    ut.rsync(src_uri, dst_uri)
    src_uri = src_uri.replace('\\', '')
    src_uri = '/'.join(src_uri.split('/')[:-2])
    ut.delete(src_uri)

    src_uri = join(ggr_path, '138', '138C/')
    dst_uri = join(ggr_path, '169', '169C/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri)

    # Conflicts - Move first

    src_uri = join(ggr_path, '115', '115A/')
    dst_uri = join(ggr_path, '191')
    ut.ensuredir(dst_uri)
    dst_uri = join(dst_uri, '191A/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    src_uri = join(ggr_path, '148', '148A/')
    dst_uri = join(ggr_path, '149', '149A-temp/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    # Conflicts - Move second

    src_uri = join(ggr_path, '117', '117A/')
    dst_uri = join(ggr_path, '115', '115A/')
    ut.rsync(src_uri, dst_uri)
    src_uri = src_uri.replace('\\', '')
    src_uri = '/'.join(src_uri.split('/')[:-2])
    ut.delete(src_uri)

    src_uri = join(ggr_path, '149', '149A/')
    dst_uri = join(ggr_path, '148', '148A/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    # Conflicts - Move third

    src_uri = join(ggr_path, '149', '149A-temp/')
    dst_uri = join(ggr_path, '149', '149A/')
    ut.rsync(src_uri, dst_uri)
    ut.delete(src_uri.replace('\\', ''))

    # Conflicts - Merge third

    src_uri = join(ggr_path, '57', '57A/')
    dst_uri = join(ggr_path, '25', '25A/')
    ut.rsync(src_uri, dst_uri)
    src_uri = src_uri.replace('\\', '')
    src_uri = '/'.join(src_uri.split('/')[:-2])
    ut.delete(src_uri)