def convert_all_to_csv(by_image=False):
"""Converts all Matlab files into corresponding CSV files."""
ifdata = dp.get_ifdata()
ifdata_train = dp.get_ifdata(split='train')
root_path = os.path.join(csv_path, 'datasets', 'stanford')
test_path = os.path.join(root_path, 'test')
train_path = os.path.join(root_path, 'train')
if by_image:
image_csv_path = os.path.join(csv_path, 'image_files')
if not os.path.exists(image_csv_path):
os.mkdir(image_csv_path)
generate_image_csvs(ifdata, image_csv_path)
else:
obj_path = os.path.join(test_path, 'obj_files')
attr_path = os.path.join(test_path, 'attr_files')
rel_path = os.path.join(test_path, 'rel_files')
class_to_idx_path = os.path.join(csv_path, 'class_to_idx.csv')
class_path = os.path.join(csv_path, 'classes.csv')
index_path = os.path.join(test_path, 'index.txt')
for path in (obj_path, attr_path, rel_path):
if not os.path.exists(path):
os.makedirs(path)
transfer_scores(ifdata, obj_path)
transfer_scores(ifdata, attr_path, is_attr=True)
transfer_rels(ifdata, rel_path)
transfer_platt_models(ifdata, obj_path, attr_path, rel_path)
transfer_class_to_idx(ifdata, class_to_idx_path)
transfer_classes(ifdata, class_path)
transfer_indices(index_path, ifdata)
generate_train_set(train_path, obj_path, attr_path, rel_path,
ifdata_train)
import os
import json
import shutil
from tqdm import tqdm
import irsg_core.data_pull as dp
from config import get_config_path
with open(get_config_path()) as f:
cfg_data = json.load(f)
sg_path = cfg_data['file_paths']['sg_path']
def copy_images_indexed(path, if_data):
from_path = os.path.join(path, 'sg_test_images')
to_path = os.path.join(path, 'by_index')
if not os.path.exists(to_path):
os.makedirs(to_path)
for index, image_data in tqdm(enumerate(if_data.vg_data),
total=len(if_data.vg_data)):
image_name = os.path.basename(image_data.image_path)
image_path_from = os.path.join(from_path, image_name)
image_path_to = os.path.join(to_path, 'i{:03d}.jpg'.format(index))
shutil.copy(image_path_from, image_path_to)
if __name__ == '__main__':
if_data = dp.get_ifdata(use_csv=True)
copy_images_indexed(sg_path, if_data)
]
bbox_dict[int(index)].append((x, y, w, h, score, -1))
data_out_path = os.path.join(output_path, data_name)
if not os.path.exists(data_out_path):
os.makedirs(data_out_path)
for index, rows in bbox_dict.iteritems():
bbox_file = 'irsg_{}.csv'.format(index)
bbox_path = os.path.join(data_out_path, bbox_file)
with open(bbox_path, 'w') as f:
csv_writer = csv.writer(f)
csv_writer.writerows(sorted(rows, key=lambda row: row[0]))
if __name__ == '__main__':
ifdata = dp.get_ifdata(use_csv=True, split='train')
in_paths = [
os.path.join(data_path, name)
for name in ('obj_rcnn_train_smol', 'obj_rcnn_val_smol')
]
out_paths = [
os.path.join(csv_path, 'datasets', *names)
for names in (('psu', 'train', 'obj_files'),
('psu', 'val', 'obj_files'), ('psu', 'test',
'obj_files'))
]
for in_path, out_path in tqdm(zip(in_paths, out_paths), desc='paths'):
convert_rcnn_data(in_path, out_path)
train_indices = data_utils.get_indices(data_path, 'train')
for query_index in range(num_queries):
csv_path = os.path.join(situate_path,
'q{:03d}_recalls.csv'.format(query_index))
try:
recall_array = np.loadtxt(csv_path, delimiter=',').reshape(-1)
except IOError:
continue
recalls[query_index] = list(recall_array)
return recalls
if __name__ == '__main__':
query_path = os.path.join(data_path, 'queries.txt')
queries = query_viz.generate_queries_from_file(query_path)
if_data = dp.get_ifdata(dataset='psu', split='test', use_csv=True)
batch_path = os.path.join(out_path, 'query_energies_psu/')
if not os.path.exists(batch_path):
os.mkdir(batch_path)
image_query_data.generate_energy_data(queries, batch_path, if_data)
geom_batch_path = os.path.join(out_path, 'query_energies_geom_psu/')
if not os.path.exists(geom_batch_path):
os.mkdir(geom_batch_path)
image_query_data.generate_energy_data(queries,
geom_batch_path,
if_data,
use_alt_geom=True)
true_geom_batch_path = os.path.join(out_path,
'query_energies_true_geom_psu/')
for obj in image_data.objects:
if data_utils.make_array(obj.names)[0] == name:
gt_bboxes.append(data_utils.make_bbox(obj.bbox))
for value in values:
ious = [
ImageQueryData.get_iou(value[:4], gt_bbox)
for gt_bbox in gt_bboxes
]
gts[name].append(any([iou >= iou_thresh for iou in ious]))
preds[name].append(value[4])
auc_scores = {}
for name in gts.keys():
rcnn_gts = gts[name]
rcnn_preds = preds[name]
auc_scores[name] = roc_auc_score(rcnn_gts, rcnn_preds)
auc_total = sum(auc_scores.values())
with open(os.path.join(csv_path, 'rcnn_weights.csv'), 'w') as f:
csv_writer = csv.writer(f)
for name, auc_score in auc_scores.iteritems():
csv_writer.writerow((name, auc_score / auc_total))
if __name__ == '__main__':
if_data = dp.get_ifdata(dataset='psu', use_csv=True, split='val')
query_path = os.path.join(data_path, 'queries.txt')
queries = query_viz.generate_queries_from_file(query_path)
# cross_validate_simple(if_data, queries, K, WEIGHTS)
cross_validate_tune_rcnns(if_data, K)
def write_obj_attr_counts(obj_count, attr_count, train_objs, train_attrs,
path):
"""Record object and attribute counts in CSV format."""
obj_path = os.path.join(path, 'obj_counts.csv')
attr_path = os.path.join(path, 'attr_counts.csv')
for csv_path, count, train_vals in ((obj_path, obj_count, train_objs),
(attr_path, attr_count, train_attrs)):
with open(csv_path, 'w') as f:
csv_writer = csv.writer(f)
csv_writer.writerow(('name', 'count', 'is_counted'))
for obj_attr, count in count.most_common():
csv_writer.writerow((obj_attr, count, obj_attr in train_vals))
def count_to_threshold_list(count, thresh):
"""Convert a Counter object into a thresholded list of items."""
return [
obj for obj, obj_count in count.most_common() if obj_count >= thresh
]
if __name__ == '__main__':
if_data_test = dp.get_ifdata(use_csv=True)
if_data_train = dp.get_ifdata(use_csv=True, split='train')
test_obj_count, test_attr_count = get_obj_attr_counts(if_data_test)
train_obj_count, train_attr_count = get_obj_attr_counts(if_data_train)
train_objs = count_to_threshold_list(train_obj_count, 50)
train_attrs = count_to_threshold_list(train_attr_count, 50)
write_obj_attr_counts(test_obj_count, test_attr_count, train_objs,
train_attrs, csv_path)