def test_all_labels():
labels = utils.read_labels()
_test_labels(labels)
_test_validate_labels(labels)
labels_directory = 'label_persons'
all_file_names = [
f for f in os.listdir(labels_directory) if f.endswith('.csv')
]
for file_name in all_file_names:
labels = utils.read_labels(os.path.join(labels_directory, file_name))
_test_labels(labels)
_test_validate_labels(labels)
def test_labels():
labels = utils.read_labels()
for label in labels:
for browser in ['firefox', 'chrome']:
assert os.path.join('data', '{}_{}.png'.format(label, browser))
import argparse
import random
from autowebcompat import network, utils
parser = argparse.ArgumentParser()
parser.add_argument('network', type=str, choices=network.SUPPORTED_NETWORKS, help='Select the network to use for training')
parser.add_argument('optimizer', type=str, choices=network.SUPPORTED_OPTIMIZERS, help='Select the optimizer to use for training')
args = parser.parse_args()
labels = utils.read_labels()
utils.prepare_images()
all_image_names = [i for i in utils.get_images() if i in labels]
all_images = sum([[i + '_firefox.png', i + '_chrome.png'] for i in all_image_names], [])
image = utils.load_image(all_images[0])
input_shape = image.shape
BATCH_SIZE = 32
EPOCHS = 50
def load_pair(fname):
f = utils.load_image(fname + '_firefox.png')
print(f.shape)
c = utils.load_image(fname + '_chrome.png')
print(c.shape)
return [f, c]
images_train = random.sample(all_image_names, int(len(all_image_names) * 0.9))
images_test = [i for i in all_image_names if i not in set(images_train)]
from autowebcompat import utils
labels_directory = 'label_persons/'
parser = argparse.ArgumentParser()
parser.add_argument('file_name',
action='store',
help='Filename to open and save your labels')
parser.add_argument('--verify',
dest='verify',
default=False,
action='store_true',
help='To verify and edit previous labels')
args = parser.parse_args()
labels = utils.read_labels(labels_directory + args.file_name + '.csv')
bounding_boxes = utils.read_bounding_boxes(labels_directory + args.file_name +
'_bounding_box.json')
if args.verify:
images_to_show = [i for i in utils.get_images() if i in labels]
else:
all_labels = utils.read_labels()
images_to_show = [i for i in utils.get_images() if i not in labels]
images_in_all_labels = [i for i in images_to_show if i in all_labels]
images_not_in_all_labels = [
i for i in images_to_show if i not in all_labels
]
random.shuffle(images_not_in_all_labels)
random.shuffle(images_in_all_labels)
images_to_show = images_not_in_all_labels + images_in_all_labels
from os import listdir
from autowebcompat import utils
labels_directory = 'label_persons/'
all_file_names = [f for f in listdir(labels_directory)]
labels_voted = {}
ydn_map = {'y': 0, 'd': 1, 'n': 2}
ydn_reverse_map = {0: 'y', 1: 'd', 2: 'n'}
for file_name in all_file_names:
labels = utils.read_labels(labels_directory + file_name)
for key, value in labels.items():
if key not in labels_voted.keys():
labels_voted[key] = [0, 0, 0]
labels_voted[key][ydn_map[value]] += 1
labels = {}
for key, values in labels_voted.items():
labels[key] = ydn_reverse_map[values.index(max(values))]
utils.write_labels(labels)
class Timer(Callback):
def on_train_begin(self, logs={}):
self.train_begin_time = time.time()
self.epoch_times = []
def on_epoch_begin(self, batch, logs={}):
self.epoch_begin_time = time.time()
def on_epoch_end(self, batch, logs={}):
self.epoch_times.append(time.time() - self.epoch_begin_time)
def on_train_end(self, logs={}):
self.train_time = time.time() - self.train_begin_time
labels = utils.read_labels(args.labels)
utils.prepare_images()
all_image_names = [i for i in utils.get_images() if i in labels]
all_images = sum([[i + '_firefox.png', i + '_chrome.png']
for i in all_image_names], [])
image = utils.load_image(all_images[0])
input_shape = image.shape
SAMPLE_SIZE = len(all_image_names)
TRAIN_SAMPLE = 80 * (SAMPLE_SIZE // 100)
VALIDATION_SAMPLE = 10 * (SAMPLE_SIZE // 100)
TEST_SAMPLE = SAMPLE_SIZE - (TRAIN_SAMPLE + VALIDATION_SAMPLE)
def load_pair(fname):
def test_read_labels():
labels = utils.read_labels(file_name='labels.csv')
assert (isinstance(labels, dict))
def migrate_v1_to_v2():
def read_sequence(bug_id):
with open('data/%d.txt' % bug_id) as f:
return f.readlines()
def write_sequence(bug_id, data):
with open('./data/%d.txt' % bug_id, 'w') as f:
for i in range(len(data)):
for j in range(i + 1):
f.write(data[j])
f.write('\n')
all_data_files = os.listdir('data')
label_files = os.listdir('label_persons')
map_names = {}
for i in range(7):
map_names[i] = str(int((i + 1) * (i + 2) / 2 - 1))
for f in all_data_files:
if '.png' not in f:
continue
parts = f.split('_')
if len(parts) <= 2:
continue
bug_id = parts[0]
seq_no = int(parts[1])
browser = parts[2]
seq_no = map_names[seq_no]
new_name = utils.create_file_name(bug_id=bug_id, browser=browser, seq_no=seq_no)
os.rename(os.path.join('data', f), os.path.join('data', new_name))
for f in all_data_files:
if '.txt' not in f:
continue
bug_id = os.path.splitext(f)[0]
data = read_sequence(int(bug_id))
write_sequence(int(bug_id), data)
for f in label_files:
if 'csv' not in f:
continue
labels = utils.read_labels(os.path.join('label_persons', f))
new_labels = {}
for key, value in labels.items():
key_info = utils.parse_file_name(key)
if 'seq_no' not in key_info:
new_labels[key] = value
continue
key = key.replace('_%d' % key_info['seq_no'], '_%s' % map_names[key_info['seq_no']])
new_labels[key] = value
utils.write_labels(new_labels, os.path.join('label_persons', f))
for f in label_files:
if 'json' not in f:
continue
bounding_boxes = utils.read_bounding_boxes(os.path.join('label_persons', f))
new_bounding_boxes = {}
for key, value in bounding_boxes.items():
parts = key.split('_')
if len(parts) <= 2:
new_bounding_boxes[key] = value
continue
bug_id = parts[0]
seq_no = int(parts[1])
browser = parts[2]
seq_no = map_names[seq_no]
key = utils.create_file_name(bug_id=bug_id, browser=browser, seq_no=seq_no)
new_bounding_boxes[key] = value
utils.write_bounding_boxes(new_bounding_boxes, os.path.join('label_persons', f))
def test_write_labels(tmpdir):
label = {'1': '1', '2': '2'}
file_path = tmpdir.join('test.csv')
utils.write_labels(label, file_name=file_path)
assert (os.path.exists(file_path))
assert (label == utils.read_labels(file_name=file_path))
def test_validate_labels():
labels = utils.read_labels()
for label in labels.values():
assert label in ['y', 'n', 'd']
