def test(gpath_list, canonical_weight_filepath=None, **kwargs):
from detecttools.directory import Directory
# Get correct weight if specified with shorthand
archive_url = None
ensemble_index = None
if canonical_weight_filepath is not None and ':' in canonical_weight_filepath:
assert canonical_weight_filepath.count(':') == 1
canonical_weight_filepath, ensemble_index = canonical_weight_filepath.split(':')
ensemble_index = int(ensemble_index)
if canonical_weight_filepath in ARCHIVE_URL_DICT:
archive_url = ARCHIVE_URL_DICT[canonical_weight_filepath]
archive_path = ut.grab_file_url(archive_url, appname='ibeis', check_hash=True)
else:
raise RuntimeError('canonical_weight_filepath %r not recognized' % (canonical_weight_filepath, ))
assert os.path.exists(archive_path)
archive_path = ut.truepath(archive_path)
ensemble_path = archive_path.strip('.zip')
if not os.path.exists(ensemble_path):
ut.unarchive_file(archive_path, output_dir=ensemble_path)
assert os.path.exists(ensemble_path)
direct = Directory(ensemble_path, include_file_extensions=['weights'], recursive=True)
weights_path_list = direct.files()
weights_path_list = sorted(weights_path_list)
assert len(weights_path_list) > 0
if ensemble_index is not None:
assert 0 <= ensemble_index and ensemble_index < len(weights_path_list)
weights_path_list = [ weights_path_list[ensemble_index] ]
assert len(weights_path_list) > 0
print('Using weights in the ensemble: %s ' % (ut.repr3(weights_path_list), ))
result_list = test_ensemble(gpath_list, weights_path_list, **kwargs)
for result in result_list:
x0 = max(result['x0'], 0.0)
y0 = max(result['y0'], 0.0)
x1 = max(result['x1'], 0.0)
y1 = max(result['y1'], 0.0)
yield (x0, y0, x1, y1, )
def __init__(pascald, dataset_path, **kwargs):
com._kwargs(kwargs, 'object_min_width', 32)
com._kwargs(kwargs, 'object_min_height', 32)
com._kwargs(kwargs, 'mine_patches', True)
com._kwargs(kwargs, 'mine_negatives', True)
com._kwargs(kwargs, 'mine_width_min', 50)
com._kwargs(kwargs, 'mine_width_max', 400)
com._kwargs(kwargs, 'mine_height_min', 50)
com._kwargs(kwargs, 'mine_height_max', 400)
com._kwargs(kwargs, 'mine_max_attempts', 100)
com._kwargs(kwargs, 'mine_max_keep', 10)
com._kwargs(kwargs, 'mine_overlap_margin', 0.25)
com._kwargs(kwargs, 'mine_exclude_categories', [])
pascald.dataset_path = dataset_path
pascald.absolute_dataset_path = os.path.realpath(dataset_path)
direct = Directory(os.path.join(pascald.dataset_path, "Annotations") , include_file_extensions=["xml"])
pascald.images = []
files = direct.files()
print("Loading Database")
for i, filename in enumerate(files):
if len(files) > 10:
if i % (len(files) / 10) == 0:
print("%0.2f" % (float(i) / len(files)))
pascald.images.append(PASCAL_Image(filename, pascald.absolute_dataset_path, **kwargs))
print(" ...Loaded")
pascald.categories_images = []
pascald.categories_rois = []
for image in pascald.images:
temp = image.categories(unique=False, patches=True)
pascald.categories_rois += temp
pascald.categories_images += set(temp)
if len(image.objects) == 0:
pascald.categories_images += ["BACKGROUND"]
pascald.distribution_images = com.histogram(pascald.categories_images)
pascald.distribution_rois = com.histogram(pascald.categories_rois)
pascald.rois = sum(pascald.distribution_rois.values())
pascald.categories = sorted(set(pascald.categories_images))
def process_image_directory(project_name, size, reset=True):
# Raw folders
raw_path = abspath(join('..', 'data', 'raw'))
processed_path = abspath(join('..', 'data', 'processed'))
# Project folders
project_raw_path = join(raw_path, project_name)
project_processed_path = join(processed_path, project_name)
# Load raw data
direct = Directory(project_raw_path, include_extensions='images')
# Reset / create paths if not exist
if exists(project_processed_path) and reset:
ut.remove_dirs(project_processed_path)
ut.ensuredir(project_processed_path)
# Process by resizing the images into the desired shape
for file_path in direct.files():
file_name = basename(file_path)
print('Processing %r' % (file_name, ))
image = cv2.imread(file_path)
image = cv2.resize(image, size, interpolation=cv2.INTER_LANCZOS4)
dest_path = join(project_processed_path, file_name)
cv2.imwrite(dest_path, image)
def train_pyrf():
# boosting = 3
num_trees = 5
category = 'zebra_grevys'
#=================================
# Train / Detect Configurations
#=================================
train_config = {
'object_min_width': 32,
'object_min_height': 32,
'mine_negatives': True,
'mine_max_keep': 1,
'mine_exclude_categories': [category],
'mine_width_min': 128,
'mine_width_max': 512,
'mine_height_min': 128,
'mine_height_max': 512,
'neg_exclude_categories': [category],
'max_rois_pos': 900,
'max_rois_neg': 550,
'num_trees': num_trees,
}
detect_config = {
'save_detection_images': True,
'percentage_top': 0.40,
}
#=================================
# Train / Detect Initialization
#=================================
# Create detector
detector = Random_Forest_Detector()
# Gather Dataset
dataset_path = utool.unixpath('~/code/IBEIS2014/')
dataset = IBEIS_Data(dataset_path, **train_config)
results_path = join(utool.unixpath('~/code/pyrf/results'), category)
# pos_path = join(results_path, 'train-positives')
# neg_path = join(results_path, 'train-negatives')
# val_path = join(results_path, 'val')
test_path = join(results_path, 'test')
# test_pos_path = join(results_path, 'test-positives')
# test_neg_path = join(results_path, 'test-negatives')
detect_path = join(results_path, 'detect')
trees_path = join(results_path, 'trees')
# # Ensure result path for the category
# # rmtreedir(results_path)
# ensuredir(results_path)
# for phase in range(1, boosting + 1):
# print("*********************")
# print("Phase: %s" % phase)
# print("*********************")
# # raw_input()
# # =================================
# # Train Random Forest
# #=================================
# detector.train(dataset, category, pos_path, neg_path, val_path,
# test_path, test_pos_path, test_neg_path,
# trees_path, reshuffle=(phase == 1), **train_config)
# if phase < boosting:
# #=================================
# # Detect using Random Forest
# #=================================
# # Load forest, so we don't have to reload every time
# forest = detector.load(trees_path, category + '-', num_trees=(phase * num_trees))
# detector.set_detect_params(**detect_config)
# # Ensure output detection paths
# rmtreedir(detect_path)
# ensuredir(detect_path)
# # Calculate error on test set
# direct = Directory(test_path, include_file_extensions=["jpg"])
# accuracy_list = []
# image_filepath_list = direct.files()
# dst_filepath_list = [ join(detect_path, split(image_filepath)[1]) for image_filepath in image_filepath_list ]
# predictions_list = detector.detect_many(forest, image_filepath_list, dst_filepath_list, use_openmp=True)
# for index, (predictions, image_filepath) in enumerate(zip(predictions_list, image_filepath_list)):
# image_path, image_filename = split(image_filepath)
# image = dataset[image_filename]
# accuracy, true_pos, false_pos, false_neg = image.accuracy(predictions, category)
# accuracy_list.append(accuracy)
# progress = "%0.2f" % (float(index) / len(image_filepath_list))
# print("TEST %s %0.4f %s" % (image, accuracy, progress), end='\r')
# sys.stdout.flush()
# # image.show(prediction_list=predictions, category=category)
# print(' ' * 100, end='\r')
# print("TEST ERROR: %0.4f" % (1.0 - (float(sum(accuracy_list)) / len(accuracy_list))))
# #=================================
# # Eval and prep boosting train set
# #=================================
# detector.boosting(phase, forest, dataset, category, pos_path, neg_path,
# test_pos_path, test_neg_path, detect_path)
####################################
# New FAST
####################################
detector = Random_Forest_Detector(
scales='6 1.0 0.75 0.55 0.40 0.30 0.20'
)
# Ensure output detection paths
detect_path_temp = detect_path + "_1"
rmtreedir(detect_path_temp)
ensuredir(detect_path_temp)
# Load forest, so we don't have to reload every time
forest = detector.load(trees_path, category + '-', num_trees=25)
detector.set_detect_params(**detect_config)
# Calculate error on test set
direct = Directory(test_path, include_file_extensions=["jpg"])
accuracy_list = []
true_pos_list = []
false_pos_list = []
false_neg_list = []
image_filepath_list = direct.files()
dst_filepath_list = [ join(detect_path_temp, split(image_filepath)[1]) for image_filepath in image_filepath_list ]
predictions_list = detector.detect_many(forest, image_filepath_list, dst_filepath_list, use_openmp=True)
for index, (predictions, image_filepath) in enumerate(zip(predictions_list, image_filepath_list)):
image_path, image_filename = split(image_filepath)
image = dataset[image_filename]
accuracy, true_pos, false_pos, false_neg = image.accuracy(predictions, category)
accuracy_list.append(accuracy)
true_pos_list.append(true_pos)
false_pos_list.append(false_pos)
false_neg_list.append(false_neg)
progress = "%0.2f" % (float(index) / len(image_filepath_list))
print("TEST %s %0.4f %s" % (image, accuracy, progress), end='\r')
sys.stdout.flush()
# image.show(prediction_list=predictions, category=category)
print(' ' * 100, end='\r')
print("1 TEST ERROR : %0.4f" % (1.0 - (float(sum(accuracy_list)) / len(accuracy_list))))
print("1 TEST TRUE POS : %d" % (sum(true_pos_list)))
print("1 TEST FALSE POS : %d" % (sum(false_pos_list)))
print("1 TEST FALSE NEG : %d" % (sum(false_neg_list)))
####################################
# New SLOW
####################################
detector = Random_Forest_Detector(
scales='11 1.5 1.25 1.0 0.8 0.64 0.51 0.41 0.33 0.26 0.21 0.17'
)
# Ensure output detection paths
detect_path_temp = detect_path + "_2"
rmtreedir(detect_path_temp)
ensuredir(detect_path_temp)
# Load forest, so we don't have to reload every time
forest = detector.load(trees_path, category + '-', num_trees=25)
detector.set_detect_params(**detect_config)
# Calculate error on test set
direct = Directory(test_path, include_file_extensions=["jpg"])
accuracy_list = []
true_pos_list = []
false_pos_list = []
false_neg_list = []
image_filepath_list = direct.files()
dst_filepath_list = [ join(detect_path_temp, split(image_filepath)[1]) for image_filepath in image_filepath_list ]
predictions_list = detector.detect_many(forest, image_filepath_list, dst_filepath_list, use_openmp=True)
for index, (predictions, image_filepath) in enumerate(zip(predictions_list, image_filepath_list)):
image_path, image_filename = split(image_filepath)
image = dataset[image_filename]
accuracy, true_pos, false_pos, false_neg = image.accuracy(predictions, category)
accuracy_list.append(accuracy)
true_pos_list.append(true_pos)
false_pos_list.append(false_pos)
false_neg_list.append(false_neg)
progress = "%0.2f" % (float(index) / len(image_filepath_list))
print("TEST %s %0.4f %s" % (image, accuracy, progress), end='\r')
sys.stdout.flush()
# image.show(prediction_list=predictions, category=category)
print(' ' * 100, end='\r')
print("2 TEST ERROR : %0.4f" % (1.0 - (float(sum(accuracy_list)) / len(accuracy_list))))
print("2 TEST TRUE POS : %d" % (sum(true_pos_list)))
print("2 TEST FALSE POS : %d" % (sum(false_pos_list)))
print("2 TEST FALSE NEG : %d" % (sum(false_neg_list)))
def numpy_processed_directory(project_name, numpy_ids_file_name='ids.npy',
numpy_x_file_name='X.npy',
numpy_y_file_name='y.npy', labels_file_name='labels.csv',
reset=True):
# Raw folders
processed_path = abspath(join('..', 'data', 'processed'))
labels_path = abspath(join('..', 'data', 'labels'))
numpy_path = abspath(join('..', 'data', 'numpy'))
# Project folders
project_processed_path = join(processed_path, project_name)
project_labels_path = join(labels_path, project_name)
project_numpy_path = join(numpy_path, project_name)
# Project files
project_numpy_ids_file_name = join(project_numpy_path, numpy_ids_file_name)
project_numpy_x_file_name = join(project_numpy_path, numpy_x_file_name)
project_numpy_y_file_name = join(project_numpy_path, numpy_y_file_name)
project_numpy_labels_file_name = join(project_labels_path, labels_file_name)
# Load raw data
direct = Directory(project_processed_path, include_extensions='images')
label_dict = {}
for line in open(project_numpy_labels_file_name):
line = line.strip().split(',')
file_name = line[0].strip()
label = line[1].strip()
label_dict[file_name] = label
# Reset / create paths if not exist
if exists(project_numpy_path) and reset:
ut.remove_dirs(project_numpy_path)
ut.ensuredir(project_numpy_path)
# Get shape for all images
shape_x = list(cv2.imread(direct.files()[0]).shape)
if len(shape_x) == 2:
shape_x = shape_x + [1]
shape_x = tuple([len(direct.files())] + shape_x[::-1]) # NOQA
shape_y = shape_x[0:1] # NOQA
# Create numpy arrays
# X = np.empty(shape_x, dtype=np.uint8)
# y = np.empty(shape_y, dtype=np.uint8)
ids = []
X = []
y = []
# Process by loading images into the numpy array for saving
for index, file_path in enumerate(direct.files()):
file_name = basename(file_path)
print('Processing %r' % (file_name, ))
image = cv2.imread(file_path)
try:
label = label_dict[file_name]
# X[index] = np.array(cv2.split(image))
# y[index] = label
# X.append(np.array(cv2.split(image))) # Lasange format
ids.append(file_name)
X.append(image) # cv2 format
y.append(label)
except KeyError:
print('Cannot find label...skipping')
# raw_input()
ids = np.array(ids)
X = np.array(X, dtype=np.uint8)
# y = np.array(y, dtype=np.uint8)
y = np.array(y)
# Save numpy array
print(' ids.shape = %r' % (ids.shape,))
print(' ids.dtype = %r' % (ids.dtype,))
print(' X.shape = %r' % (X.shape,))
print(' X.dtype = %r' % (X.dtype,))
print(' y.shape = %r' % (y.shape,))
print(' y.dtype = %r' % (y.dtype,))
np.save(project_numpy_ids_file_name, ids)
np.save(project_numpy_x_file_name, X)
np.save(project_numpy_y_file_name, y)
def numpy_processed_directory5(extracted_path, numpy_ids_file_name='ids.npy',
numpy_x_file_name='X.npy',
numpy_y_file_name='y.npy',
labels_file_name='labels.csv',
reset=True, verbose=False):
print('Caching images into Numpy files with category vector...')
raw_path = join(extracted_path, 'raw')
labels_path = join(extracted_path, 'labels')
# Project files
project_numpy_ids_file_name = join(raw_path, numpy_ids_file_name)
project_numpy_x_file_name = join(raw_path, numpy_x_file_name)
project_numpy_y_file_name = join(labels_path, numpy_y_file_name)
project_numpy_labels_file_name = join(labels_path, labels_file_name)
# Load raw data
direct = Directory(raw_path, include_extensions='images')
label_dict = {}
for line in open(project_numpy_labels_file_name):
line = line.strip().split(',')
file_name = line[0].strip()
label = line[1].strip()
label_list = label.split(';')
label_list = [
list(map(float, _.split('^')))
for _ in label_list
]
label = np.array(label_list)
label_dict[file_name] = label
# Create numpy arrays
ids = []
X = []
y = []
# Process by loading images into the numpy array for saving
for index, file_path in enumerate(direct.files()):
file_name = basename(file_path)
if verbose:
print('Processing %r' % (file_name, ))
image = cv2.imread(file_path, -1)
try:
label = label_dict[file_name]
ids.append(file_name)
X.append(image)
y.append(label)
except KeyError:
print('Cannot find label...skipping')
ids = np.array(ids)
X = np.array(X, dtype=np.uint8)
y = np.array(y)
# Save numpy array
print(' ids.shape = %r' % (ids.shape,))
print(' ids.dtype = %r' % (ids.dtype,))
print(' X.shape = %r' % (X.shape,))
print(' X.dtype = %r' % (X.dtype,))
print(' y.shape = %r' % (y.shape,))
print(' y.dtype = %r' % (y.dtype,))
np.save(project_numpy_ids_file_name, ids)
np.save(project_numpy_x_file_name, X)
np.save(project_numpy_y_file_name, y)
return project_numpy_ids_file_name, project_numpy_x_file_name, project_numpy_y_file_name
def numpy_processed_directory3(extracted_path, numpy_ids_file_name='ids.npy',
numpy_x_file_name='X.npy',
numpy_y_file_name='y.npy',
labels_file_name='labels.csv',
categories_file_name='categories.csv',
reset=True, verbose=False):
print('Caching images into Numpy files with category vector...')
raw_path = join(extracted_path, 'raw')
labels_path = join(extracted_path, 'labels')
# Project files
project_numpy_ids_file_name = join(raw_path, numpy_ids_file_name)
project_numpy_x_file_name = join(raw_path, numpy_x_file_name)
project_numpy_y_file_name = join(labels_path, numpy_y_file_name)
project_numpy_labels_file_name = join(labels_path, labels_file_name)
project_numpy_categories_file_name = join(labels_path, categories_file_name)
category_list = []
for line in open(project_numpy_categories_file_name):
category = line.strip()
if len(category) > 0:
category_list.append(category)
# Load raw data
direct = Directory(raw_path, include_extensions='images')
label_dict = {}
count_dict = {}
for line in open(project_numpy_labels_file_name):
line = line.strip().split(',')
file_name = line[0].strip()
label = line[1].strip()
label_list = label.split(';')
label_set = set(label_list)
label = [ 1 if category_ in label_set else 0 for category_ in category_list]
assert 1 in label
count = label.count(1)
if count not in count_dict:
count_dict[count] = 0
count_dict[count] += 1
label_dict[file_name] = label
print('count_dict = %s' % (ut.repr3(count_dict), ))
# Get shape for all images
shape_x = list(cv2.imread(direct.files()[0]).shape)
if len(shape_x) == 2:
shape_x = shape_x + [1]
shape_x = tuple([len(direct.files())] + shape_x[::-1]) # NOQA
shape_y = shape_x[0:1] # NOQA
# Create numpy arrays
# X = np.empty(shape_x, dtype=np.uint8)
# y = np.empty(shape_y, dtype=np.uint8)
ids = []
X = []
y = []
# Process by loading images into the numpy array for saving
for index, file_path in enumerate(direct.files()):
file_name = basename(file_path)
if verbose:
print('Processing %r' % (file_name, ))
image = cv2.imread(file_path)
try:
label = np.array(label_dict[file_name])
# X[index] = np.array(cv2.split(image))
# y[index] = label
# X.append(np.array(cv2.split(image))) # Lasange format
ids.append(file_name)
X.append(image) # cv2 format
y.append(label)
except KeyError:
print('Cannot find label...skipping')
# raw_input()
ids = np.array(ids)
X = np.array(X, dtype=np.uint8)
# y = np.array(y, dtype=np.uint8)
y = np.vstack(y)
# Save numpy array
print(' ids.shape = %r' % (ids.shape,))
print(' ids.dtype = %r' % (ids.dtype,))
print(' X.shape = %r' % (X.shape,))
print(' X.dtype = %r' % (X.dtype,))
print(' y.shape = %r' % (y.shape,))
print(' y.dtype = %r' % (y.dtype,))
print(' categories = %r' % (category_list,))
np.save(project_numpy_ids_file_name, ids)
np.save(project_numpy_x_file_name, X)
np.save(project_numpy_y_file_name, y)
return project_numpy_ids_file_name, project_numpy_x_file_name, project_numpy_y_file_name
activity_csv_fpath = join(csv_fpath, 'Group-Habitat-Activity table.csv')
exportedmdb_fpath = join(csv_fpath, 'Individual sightings.csv')
utool.checkpath(activity_csv_fpath, verbose=True)
utool.checkpath(exportedmdb_fpath, verbose=True)
with open(join(activity_csv_fpath), 'r') as file_:
lines = file_.read()
for line in lines.splitlines()[1:]:
line = [ item.strip() for item in line.strip().split(',')]
_id = line[2]
if _id not in activities:
activities[_id] = [line[col] for col in columns]
originals = join(prefix, 'Ol_pejeta_zebra_photos2__1GB')
images = Directory(originals)
image_set = set(images.files())
print(images)
exts = []
for image in images.files():
exts.append(image.split('.')[-1])
exts = list(set(exts))
print('EXTENSIONS: %r ' % (exts,))
print('''
=====================
PROCESSING IMAGESETS
=====================
''')
used = []
# e ncounters = open(join(prefix, 'e ncounters.csv'),'w')
def test(gpath_list,
classifier_weight_filepath=None,
return_dict=False,
multiclass=False,
**kwargs):
from detecttools.directory import Directory
# Get correct weight if specified with shorthand
archive_url = None
ensemble_index = None
if classifier_weight_filepath is not None and ':' in classifier_weight_filepath:
assert classifier_weight_filepath.count(':') == 1
classifier_weight_filepath, ensemble_index = classifier_weight_filepath.split(
':')
ensemble_index = int(ensemble_index)
if classifier_weight_filepath in ARCHIVE_URL_DICT:
archive_url = ARCHIVE_URL_DICT[classifier_weight_filepath]
archive_path = ut.grab_file_url(archive_url,
appname='ibeis',
check_hash=True)
else:
print('classifier_weight_filepath %r not recognized' %
(classifier_weight_filepath, ))
raise RuntimeError
assert os.path.exists(archive_path)
archive_path = ut.truepath(archive_path)
ensemble_path = archive_path.strip('.zip')
if not os.path.exists(ensemble_path):
ut.unarchive_file(archive_path, output_dir=ensemble_path)
assert os.path.exists(ensemble_path)
direct = Directory(ensemble_path,
include_file_extensions=['weights'],
recursive=True)
weights_path_list = direct.files()
weights_path_list = sorted(weights_path_list)
assert len(weights_path_list) > 0
kwargs.pop('classifier_algo', None)
print('Using weights in the ensemble, index %r: %s ' % (
ensemble_index,
ut.repr3(weights_path_list),
))
result_list = test_ensemble(gpath_list,
weights_path_list,
classifier_weight_filepath,
ensemble_index,
multiclass=multiclass,
**kwargs)
for result in result_list:
best_key = None
best_score = -1.0
for key, score in result.items():
if score > best_score:
best_key = key
best_score = score
assert best_score >= 0.0 and best_key is not None
if return_dict:
yield best_score, best_key, result
else:
yield best_score, best_key
def train(rf, train_pos_cpath_list, train_neg_cpath_list, trees_path, **kwargs):
"""
Train a new forest with the given positive chips and negative chips.
Args:
train_pos_chip_path_list (list of str): list of positive training chips
train_neg_chip_path_list (list of str): list of negative training chips
trees_path (str): string path of where the newly trained trees are to be saved
Kwargs:
chips_norm_width (int, optional): Chip normalization width for resizing;
the chip is resized to have a width of chips_norm_width and
whatever resulting height in order to best match the original
aspect ratio; defaults to 128
If both chips_norm_width and chips_norm_height are specified,
the original aspect ratio of the chip is not respected
chips_norm_height (int, optional): Chip normalization height for resizing;
the chip is resized to have a height of chips_norm_height and
whatever resulting width in order to best match the original
aspect ratio; defaults to None
If both chips_norm_width and chips_norm_height are specified,
the original aspect ratio of the chip is not respected
chips_prob_flip_horizontally (float, optional): The probability
that a chips is flipped horizontally before training to make
the training set invariant to horizontal flips in the image;
defaults to 0.5; 0.0 <= chips_prob_flip_horizontally <= 1.0
chips_prob_flip_vertically (float, optional): The probability
that a chips is flipped vertivcally before training to make
the training set invariant to vertical flips in the image;
defaults to 0.5; 0.0 <= chips_prob_flip_vertically <= 1.0
patch_width (int, optional): the width of the patches for extraction
in the tree; defaults to 32; patch_width > 0
patch_height (int, optional): the height of the patches for extraction
in the tree; defaults to 32; patch_height > 0
patch_density (float, optional): the number of patches to extract from
each chip as a function of density; the density is calculated as:
samples = patch_density * [(chip_width * chip_height) / (patch_width * patch_height)]
and specifies how many times a particular pixel is sampled
from the chip; defaults to 4.0; patch_density > 0
trees_num (int, optional): the number of trees to train in parallel;
defaults to 10
trees_offset (int, optional): the tree number that begins the sequence
of when a tree is trained; defaults to None
If None is specified, the trees_offset value is automatically guessed
by using the number of files in trees_path
Tree model files are overwritten if the offset has overlap with
previouly generated trees
trees_max_depth (int, optional): the maximum depth of the tree during
training, this can used for regularization; defaults to 16
trees_max_patches (int, optional): the maximum number of patches that
should be extracted for training between positives AND negatives
(the detector attempts to balance between the number of positive
and negative patches to be roughly the same in quantity);
defaults to 64000
trees_leaf_size (int, optional): the number of patches in a node that
specifies the threshold for becoming a leaf; defaults to 20
A node becomes a leaf under two conditions:
1.) The maximum tree depth has been reached (trees_max_depth)
2.) The patches in the node is less than trees_leaf_size and
is stopped prematurely
trees_pixel_tests (int, optional): the number of pixel tests to perform
at each node; defaults to 2000
trees_prob_optimize_mode (float, optional): The probability of the
tree optimizing between classification and regression; defaults to
0.5
serial (bool, optional): flag to signify if to run training in serial;
defaults to False
verbose (bool, optional): verbose flag; defaults to object's verbose or
selectively enabled for this function
Returns:
None
"""
# Default values
params = odict([
('chips_norm_width', 128),
('chips_norm_height', None),
('chips_prob_flip_horizontally', 0.5),
('chips_prob_flip_vertically', 0.0),
('patch_width', 32),
('patch_height', 32),
('patch_density', 4.0),
('trees_num', 10),
('trees_offset', None),
('trees_max_depth', 16),
('trees_max_patches', 64000),
('trees_leaf_size', 20),
('trees_pixel_tests', 10000),
('trees_prob_optimize_mode', 0.5),
('serial', False),
('verbose', rf.verbose),
('quiet', rf.quiet),
])
#params.update(kwargs)
ut.update_existing(params, kwargs)
# Make the tree path absolute
trees_path = abspath(trees_path)
# cout << "AIM FOR A SPLIT OF 24k - 32k POSITIVE & NEGATIVE PATCHES EACH FOR GOOD REGULARIZATION AT DEPTH 16" << endl;
# Ensure the trees_path exists
ut.ensuredir(trees_path)
data_path = join(trees_path, 'data')
if isdir(data_path):
shutil.rmtree(data_path)
ut.ensuredir(data_path)
data_path_pos = join(data_path, 'pos')
ut.ensuredir(data_path_pos)
data_path_neg = join(data_path, 'neg')
ut.ensuredir(data_path_neg)
# Try to figure out the correct tree offset
if params['trees_offset'] is None:
direct = Directory(trees_path, include_file_extensions=['txt'])
params['trees_offset'] = len(direct.files()) + 1
if not params['quiet']:
print('[pyrf py] Auto Tree Offset: %d' % params['trees_offset'])
# Data integrity
assert params['chips_norm_width'] is None or params['chips_norm_width'] >= params['patch_width'], \
'Normalization width too small for patch width'
assert params['chips_norm_height'] is None or params['chips_norm_height'] >= params['patch_height'], \
'Normalization height too small for patch height'
assert params['patch_width'] > 0, \
'Patch width must be positive'
assert params['patch_height'] > 0, \
'Patch height must be positive'
assert params['patch_density'] > 0.0, \
'Patch density must be positive'
assert 0.0 <= params['chips_prob_flip_horizontally'] and params['chips_prob_flip_horizontally'] <= 1.0, \
'Horizontal flip probability must be between 0 and 1'
assert 0.0 <= params['chips_prob_flip_vertically'] and params['chips_prob_flip_vertically'] <= 1.0, \
'Vertical flip probability must be between 0 and 1'
assert params['trees_num'] > 0, \
'Number of trees must be positive'
assert params['trees_offset'] >= 0, \
'Number of trees must be non-negative'
assert params['trees_max_depth'] > 1, \
'Tree depth must be greater than 1'
assert params['trees_max_patches'] % 2 == 0 and params['trees_max_patches'] > 0, \
'A tree must have an even (positive) number of patches'
assert 0.0 <= params['trees_prob_optimize_mode'] and params['trees_prob_optimize_mode'] <= 1.0, \
'Tree optimization mode probability must be between 0 and 1 (inclusive)'
assert all( [ exists(train_pos_cpath) for train_pos_cpath in train_pos_cpath_list ] ), \
'At least one specified positive chip path does not exist'
assert all( [ exists(train_neg_cpath) for train_neg_cpath in train_neg_cpath_list ] ), \
'At least one specified positive chip path does not exist'
# We will let the C++ code perform the patch size checks
if not params['quiet']:
print('[pyrf py] Caching positives into %r' % (data_path_pos, ))
train_pos_chip_filename_list = _cache_data(train_pos_cpath_list, data_path_pos, **params)
if not params['quiet']:
print('[pyrf py] Caching negatives into %r' % (data_path_neg, ))
train_neg_chip_filename_list = _cache_data(train_neg_cpath_list, data_path_neg, **params)
# We no longer need these parameters (and they should not be transferred to the C++ library)
del params['chips_norm_width']
del params['chips_norm_height']
del params['chips_prob_flip_horizontally']
del params['chips_prob_flip_vertically']
# Run training algorithm
params_list = [
data_path_pos,
_cast_list_to_c(ensure_bytes_strings(train_pos_chip_filename_list), C_CHAR),
len(train_pos_chip_filename_list),
data_path_neg,
_cast_list_to_c(ensure_bytes_strings(train_neg_chip_filename_list), C_CHAR),
len(train_neg_chip_filename_list),
trees_path,
] + list(params.values())
RF_CLIB.train(rf.detector_c_obj, *params_list)
if not params['quiet']:
print('\n\n[pyrf py] *************************************')
print('[pyrf py] Training Completed')
def train_folder(rf, train_pos_path, train_neg_path, trees_path, **kwargs):
direct = Directory(train_pos_path, include_file_extensions='images')
train_pos_cpath_list = direct.files()
direct = Directory(train_neg_path, include_file_extensions='images')
train_neg_cpath_list = direct.files()
return rf.train(train_pos_cpath_list, train_neg_cpath_list, trees_path, **kwargs)
activity_csv_fpath = join(csv_fpath, 'Group-Habitat-Activity table.csv')
exportedmdb_fpath = join(csv_fpath, 'Individual sightings.csv')
utool.checkpath(activity_csv_fpath, verbose=True)
utool.checkpath(exportedmdb_fpath, verbose=True)
with open(join(activity_csv_fpath), 'r') as file_:
lines = file_.read()
for line in lines.splitlines()[1:]:
line = [ item.strip() for item in line.strip().split(',')]
_id = line[2]
if _id not in activities:
activities[_id] = [line[col] for col in columns]
originals = join(prefix, 'Ol_pejeta_zebra_photos2__1GB')
images = Directory(originals)
image_set = set(images.files())
print(images)
exts = []
for image in images.files():
exts.append(image.split('.')[-1])
exts = list(set(exts))
print('EXTENSIONS: %r ' % (exts,))
print('''
=====================
PROCESSING ENCOUNTERS
=====================
''')
used = []
# encounters = open(join(prefix, 'encounters.csv'),'w')
def status():
cars = {}
# STEP 1 - GPS
try:
gps_path = join(DEFAULT_DATA_DIR, 'gps')
direct = Directory(gps_path,
recursive=True,
include_file_extensions=[])
for direct_car in direct.directory_list:
base_car = direct_car.base()
car_path = join(gps_path, base_car)
# Establish flags
flags = {}
flags['submitted_gpx'] = exists(join(car_path, 'track.gpx'))
flags['submitted_generated_json'] = exists(
join(car_path, 'track.json'))
cars[base_car] = {'gps': flags}
cars[base_car]['persons'] = {}
except:
print("No GPS data submitted yet")
# STEP 2 - IMAGES
try:
images_path = join(DEFAULT_DATA_DIR, 'images')
direct = Directory(images_path,
recursive=True,
include_file_extensions=[])
for direct_car in direct.directory_list:
base_car = direct_car.base()
if base_car not in cars:
cars[base_car] = {}
cars[base_car]['persons'] = {}
# Construct dict for person
for direct_person in direct_car.directory_list:
base_person = direct_person.base()
person_path = join(images_path, base_car, base_person)
# Establish flags, inherit GPS for person if car has GPS flags
if 'gps' in cars[base_car]:
flags = dict(cars[base_car]['gps'])
else:
flags = {}
flags['submitted_images'] = True
flags['submitted_first'] = exists(
join(person_path, 'first.jpg'))
flags['submitted_last'] = exists(join(person_path, 'last.jpg'))
# Giraffe flags
giraffe_path = join(person_path, 'giraffe')
if isdir(giraffe_path):
flags['submitted_giraffe'] = True
temp = Directory(giraffe_path,
include_file_extensions='images')
flags['submitted_giraffe_number'] = len(temp.files())
# Zebra flags
zebra_path = join(person_path, 'zebra')
if isdir(zebra_path):
flags['submitted_zebra'] = True
temp = Directory(zebra_path,
include_file_extensions='images')
flags['submitted_zebra_number'] = len(temp.files())
# Generated offset
offset_path = join(person_path, 'offset.json')
if exists(offset_path):
flags['submitted_generated_offset'] = True
with open(offset_path, 'r') as off:
data = json.load(off)
offset = data.get('offset', 0.0)
flags['submitted_generated_offset_number'] = offset
# Assign flags to person
cars[base_car]['persons'][base_person] = flags
cars[base_car]['persons_number'] = len(
cars[base_car]['persons'].keys())
except:
print("No Image data submitted yet")
# STEP 3 - ANALYSIS
try:
analysis_path = join(DEFAULT_DATA_DIR, 'analysis')
direct = Directory(analysis_path,
recursive=True,
include_file_extensions=[])
for direct_car in direct.directory_list:
base_car = direct_car.base()
if base_car not in cars:
cars[base_car] = {}
# Append to dict for person
for direct_person in direct_car.directory_list:
base_person = direct_person.base()
person_path = join(analysis_path, base_car, base_person)
# Establish flags
if base_person in cars[base_car]['persons']:
flags = dict(cars[base_car]['persons'][base_person])
else:
flags = {}
flags['analyzed'] = True
flags['analyzed_review'] = exists(
join(person_path, 'review.flag'))
# Giraffe flags
giraffe_path = join(person_path, 'giraffe')
if isdir(giraffe_path):
flags['analyzed_giraffe'] = True
flags['analyzed_generated_giraffe_confidences'] = exists(
join(giraffe_path, 'confidences.json'))
temp = Directory(giraffe_path,
include_file_extensions=['jpg', 'json'])
flags['analyzed_giraffe_number'] = (
len(temp.files()) - 1
) / 4.0 # 4 images per detection generated (minus 1 confidences)
if not flags[
'analyzed_generated_giraffe_confidences'] and flags[
'analyzed_giraffe_number'] == -0.25:
flags['analyzed_giraffe_number'] = 0.0
# Zebra flags
zebra_path = join(person_path, 'zebra')
if isdir(zebra_path):
flags['analyzed_zebra'] = True
flags['analyzed_generated_zebra_confidences'] = exists(
join(zebra_path, 'confidences.json'))
temp = Directory(zebra_path,
include_file_extensions=['jpg', 'json'])
flags['analyzed_zebra_number'] = (
len(temp.files()) - 1
) / 4.0 # 4 images per detection generated (minus 1 confidences)
if not flags[
'analyzed_generated_zebra_confidences'] and flags[
'analyzed_zebra_number'] == -0.25:
flags['analyzed_zebra_number'] = 0.0
# Assign flags to person
cars[base_car]['persons'][base_person] = flags
cars[base_car]['persons_number'] = len(
cars[base_car]['persons'].keys())
except:
print("No images analyzed yet")
# STEP 4 - PDFS
try:
pdfs_path = join(DEFAULT_DATA_DIR, 'pdfs')
direct = Directory(pdfs_path,
recursive=True,
include_file_extensions=[])
for direct_car in direct.directory_list:
base_car = direct_car.base()
if base_car not in cars:
cars[base_car] = {}
# Append to dict for person
for direct_person in direct_car.directory_list:
base_person = direct_person.base()
person_path = join(pdfs_path, base_car, base_person)
# Establish flags
if base_person in cars[base_car]['persons']:
flags = dict(cars[base_car]['persons'][base_person])
else:
flags = {}
flags['reviewed'] = True
flags['reviewed_posted'] = exists(
join(person_path, 'content.html'))
flags['reviewed_rendered'] = exists(
join(person_path, 'content.pdf'))
flags['reviewed_printed'] = exists(
join(person_path, 'printed.flag'))
# Assign flags to person
cars[base_car]['persons'][base_person] = flags
cars[base_car]['persons_number'] = len(
cars[base_car]['persons'].keys())
except:
print("No PDFs rendered yet")
extra = {}
extra['cars'] = cars
extra['cars_number'] = len(cars.keys())
return sf.response(**extra)
def convert_ggr2018_to_ibeis(ggr_path, dbdir=None, purge=True, dry_run=False,
apply_updates=True, **kwargs):
r"""Convert the raw GGR2 (2018) data to an ibeis database.
Args
ggr_path (str): Directory to folder *containing* raw GGR 2018 data
dbdir (str): Output directory
CommandLine:
python -m ibeis convert_ggr2018_to_ibeis
Example:
>>> # SCRIPT
>>> from ibeis.dbio.ingest_ggr import * # NOQA
>>> default_ggr_path = join('/', 'data', 'ibeis', 'GGR2', 'GGR2018data')
>>> default_dbdir = join('/', 'data', 'ibeis', 'GGR2-IBEIS')
>>> dbdir = ut.get_argval('--dbdir', type_=str, default=default_dbdir)
>>> ggr_path = ut.get_argval('--ggr', type_=str, default=default_ggr_path)
>>> result = convert_ggr2018_to_ibeis(ggr_path, dbdir=dbdir, purge=False, dry_run=True, apply_updates=False)
>>> print(result)
"""
ALLOWED_NUMBERS = list(range(1, 250))
ALLOWED_LETTERS = ['A', 'B', 'C', 'D', 'E', 'F']
################################################################################
if apply_updates:
_fix_ggr2018_directory_structure(ggr_path)
################################################################################
blacklist_filepath_set = set([
join(ggr_path, 'Cameras info.numbers'),
join(ggr_path, 'Cameras info.xlsx'),
join(ggr_path, 'GGR_photos_MRC_29.1.18.ods'),
join(ggr_path, 'Cameras info-2.numbers'),
])
# Check root files
direct = Directory(ggr_path)
for filepath in direct.files(recursive=False):
try:
assert filepath in blacklist_filepath_set
ut.delete(filepath)
except AssertionError:
print('Unresolved root file found in %r' % (filepath, ))
continue
################################################################################
if purge:
ut.delete(dbdir)
ibs = ibeis.opendb(dbdir=dbdir)
################################################################################
# Check folder structure
assert exists(ggr_path)
direct = Directory(ggr_path, recursive=0)
direct1_list = direct.directories()
direct1_list.sort(key=lambda x: int(x.base()), reverse=False)
for direct1 in direct1_list:
if not dry_run:
print('Processing directory: %r' % (direct1, ))
base1 = direct1.base()
try:
int(base1)
except ValueError:
print('Error found in %r' % (direct1, ))
continue
try:
assert len(direct1.files(recursive=False)) == 0
except AssertionError:
print('Files found in %r' % (direct1, ))
continue
seen_letter_list = []
direct1_ = Directory(direct1.absolute_directory_path, recursive=0)
direct2_list = direct1_.directories()
direct2_list.sort(key=lambda x: x.base(), reverse=False)
for direct2 in direct2_list:
base2 = direct2.base()
try:
assert base2.startswith(base1)
except AssertionError:
print('Folder name heredity conflict %r with %r' % (direct2, direct1, ))
continue
try:
assert len(base2) >= 2
assert ' ' not in base2
number = base2[:-1]
letter = base2[-1]
number = int(number)
letter = letter.upper()
assert number in ALLOWED_NUMBERS
assert letter in ALLOWED_LETTERS
seen_letter_list.append(letter)
except ValueError:
print('Error found in %r' % (direct2, ))
continue
except AssertionError:
print('Folder name format error found in %r' % (direct2, ))
continue
direct2_ = Directory(direct2.absolute_directory_path, recursive=True, images=True)
try:
assert len(direct2_.directories()) == 0
except AssertionError:
print('Folders exist in file only level %r' % (direct2, ))
continue
filepath_list = sorted(direct2_.files())
if not dry_run:
try:
gid_list = ibs.add_images(filepath_list)
gid_list = ut.filter_Nones(gid_list)
gid_list = sorted(list(set(gid_list)))
imageset_text = 'GGR2,%d,%s' % (number, letter, )
note_list = [
'%s,%05d' % (imageset_text, index + 1)
for index, gid in enumerate(gid_list)
]
ibs.set_image_notes(gid_list, note_list)
ibs.set_image_imagesettext(gid_list, [imageset_text] * len(gid_list))
except Exception as ex: # NOQA
ut.embed()
seen_letter_set = set(seen_letter_list)
try:
assert len(seen_letter_set) == len(seen_letter_list)
except AssertionError:
print('Duplicate letters in %r with letters %r' % (direct1, seen_letter_list, ))
continue
try:
assert 'A' in seen_letter_set
except AssertionError:
print('WARNING: A camera not found in %r' % (direct1, ))
continue
return ibs
def _fix_ggr2018_directory_structure(ggr_path):
# Manual fixes for bad directories
src_uri = join(ggr_path, 'Clarine\ Plane\ Kurungu/')
dst_uri = join(ggr_path, '231/')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '231B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, 'Alex\ Peltier\ -\ Plane\ -\ Ngurnit/giraffe\ grevy\ count\ feb\ 18/')
dst_uri = join(ggr_path, '232/')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '232B/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)
src_uri = join(ggr_path, 'Mint\ Media\ Footage', 'Mpala\ day\ 1\ spark', 'PANORAMA/')
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, 'Mint\ Media\ Footage', 'Mpala\ day\ 1/')
dst_uri = join(ggr_path, '233/')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '233B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, 'Mint\ Media\ Footage', 'Mpala\ day\ 1\ spark/')
dst_uri = join(ggr_path, '233', '233B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, 'Mint\ Media\ Footage', 'Mpala\ day2\ /')
dst_uri = join(ggr_path, '233', '233B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, 'Mint\ Media\ Footage', 'Mpala\ day\ 2\ spark/')
dst_uri = join(ggr_path, '233', '233B/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)
src_uri = join(ggr_path, '103\ \(1\)/')
dst_uri = join(ggr_path, '103/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '103\ \(ccef473b\)/')
dst_uri = join(ggr_path, '103/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '108\ \(1\)/')
dst_uri = join(ggr_path, '108/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '226A\ \(Shaba\ Funan\ Camp\)/')
dst_uri = join(ggr_path, '226/')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '226A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '121/*.*')
dst_uri = join(ggr_path, '121', '121A/')
ut.rsync(src_uri, dst_uri)
for src_filepath in ut.glob(src_uri.replace('\\', '')):
ut.delete(src_filepath)
src_uri = join(ggr_path, '54', '54A\(16\)/')
dst_uri = join(ggr_path, '54', '54A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '54', '54B\(16\)/')
dst_uri = join(ggr_path, '54', '54B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '87', '87/')
dst_uri = join(ggr_path, '87', '87A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '223', 'A/')
dst_uri = join(ggr_path, '223', '223A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '223', 'B/')
dst_uri = join(ggr_path, '223', '223B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '14', '15A/')
dst_uri = join(ggr_path, '14', '14A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '73/')
dst_uri = join(ggr_path, '85/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '117', '115A/')
dst_uri = join(ggr_path, '117', '117A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '200\ A/')
dst_uri = join(ggr_path, '200', '200A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '200\ B/')
dst_uri = join(ggr_path, '200', '200B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '200\ F/')
dst_uri = join(ggr_path, '200', '200F/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '200A/')
dst_uri = join(ggr_path, '201/')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '201A/')
ut.rsync(src_uri, dst_uri)
# ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '201\ E/')
dst_uri = join(ggr_path, '201', '201E/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '201\ F/')
dst_uri = join(ggr_path, '201', '201F/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '200A/')
dst_uri = join(ggr_path, '202/')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '202A/')
ut.rsync(src_uri, dst_uri)
# ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '202\ B/')
dst_uri = join(ggr_path, '202', '202B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '200', '202\ F/')
dst_uri = join(ggr_path, '202', '202F/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '230', '230A', 'El\ Karama/*.*')
dst_uri = join(ggr_path, '230', '230A/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-1])
ut.delete(src_uri)
src_uri = join(ggr_path, '136', '136B', '136B\ Grevys\ Rally/*.*')
dst_uri = join(ggr_path, '136', '136B/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-1])
ut.delete(src_uri)
src_uri = join(ggr_path, '160', '160E', '104DUSIT')
if exists(src_uri):
direct = Directory(src_uri, recursive=False)
filename_list = direct.files()
for filename in sorted(filename_list):
dst_uri = filename.replace('104DUSIT/', '').replace('.JPG', '_.JPG')
assert not exists(dst_uri)
ut.rsync(filename, dst_uri)
ut.delete(src_uri)
src_uri = join(ggr_path, '222', '222B', '102DUSIT')
if exists(src_uri):
direct = Directory(src_uri, recursive=False)
filename_list = direct.files()
for filename in sorted(filename_list):
dst_uri = filename.replace('102DUSIT/', '').replace('.JPG', '_.JPG')
assert not exists(dst_uri)
ut.rsync(filename, dst_uri)
ut.delete(src_uri)
# Errors found by QR codes
# No conflicts
src_uri = join(ggr_path, '5', '5A/')
dst_uri = join(ggr_path, '5', '5B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '14', '14A/')
dst_uri = join(ggr_path, '14', '14B/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '118', '118A/')
dst_uri = join(ggr_path, '192')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '192A/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)
src_uri = join(ggr_path, '119', '119A/')
dst_uri = join(ggr_path, '189')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '189A/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)
src_uri = join(ggr_path, '120', '120A/')
dst_uri = join(ggr_path, '190')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '190A/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)
src_uri = join(ggr_path, '138', '138C/')
dst_uri = join(ggr_path, '169', '169C/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri)
# Conflicts - Move first
src_uri = join(ggr_path, '115', '115A/')
dst_uri = join(ggr_path, '191')
ut.ensuredir(dst_uri)
dst_uri = join(dst_uri, '191A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
src_uri = join(ggr_path, '148', '148A/')
dst_uri = join(ggr_path, '149', '149A-temp/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
# Conflicts - Move second
src_uri = join(ggr_path, '117', '117A/')
dst_uri = join(ggr_path, '115', '115A/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)
src_uri = join(ggr_path, '149', '149A/')
dst_uri = join(ggr_path, '148', '148A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
# Conflicts - Move third
src_uri = join(ggr_path, '149', '149A-temp/')
dst_uri = join(ggr_path, '149', '149A/')
ut.rsync(src_uri, dst_uri)
ut.delete(src_uri.replace('\\', ''))
# Conflicts - Merge third
src_uri = join(ggr_path, '57', '57A/')
dst_uri = join(ggr_path, '25', '25A/')
ut.rsync(src_uri, dst_uri)
src_uri = src_uri.replace('\\', '')
src_uri = '/'.join(src_uri.split('/')[:-2])
ut.delete(src_uri)