def test_all_labels(): labels = utils.read_labels() _test_labels(labels) _test_validate_labels(labels) labels_directory = 'label_persons' all_file_names = [ f for f in os.listdir(labels_directory) if f.endswith('.csv') ] for file_name in all_file_names: labels = utils.read_labels(os.path.join(labels_directory, file_name)) _test_labels(labels) _test_validate_labels(labels)
def test_labels(): labels = utils.read_labels() for label in labels: for browser in ['firefox', 'chrome']: assert os.path.join('data', '{}_{}.png'.format(label, browser))
import argparse import random from autowebcompat import network, utils parser = argparse.ArgumentParser() parser.add_argument('network', type=str, choices=network.SUPPORTED_NETWORKS, help='Select the network to use for training') parser.add_argument('optimizer', type=str, choices=network.SUPPORTED_OPTIMIZERS, help='Select the optimizer to use for training') args = parser.parse_args() labels = utils.read_labels() utils.prepare_images() all_image_names = [i for i in utils.get_images() if i in labels] all_images = sum([[i + '_firefox.png', i + '_chrome.png'] for i in all_image_names], []) image = utils.load_image(all_images[0]) input_shape = image.shape BATCH_SIZE = 32 EPOCHS = 50 def load_pair(fname): f = utils.load_image(fname + '_firefox.png') print(f.shape) c = utils.load_image(fname + '_chrome.png') print(c.shape) return [f, c] images_train = random.sample(all_image_names, int(len(all_image_names) * 0.9)) images_test = [i for i in all_image_names if i not in set(images_train)]
from autowebcompat import utils labels_directory = 'label_persons/' parser = argparse.ArgumentParser() parser.add_argument('file_name', action='store', help='Filename to open and save your labels') parser.add_argument('--verify', dest='verify', default=False, action='store_true', help='To verify and edit previous labels') args = parser.parse_args() labels = utils.read_labels(labels_directory + args.file_name + '.csv') bounding_boxes = utils.read_bounding_boxes(labels_directory + args.file_name + '_bounding_box.json') if args.verify: images_to_show = [i for i in utils.get_images() if i in labels] else: all_labels = utils.read_labels() images_to_show = [i for i in utils.get_images() if i not in labels] images_in_all_labels = [i for i in images_to_show if i in all_labels] images_not_in_all_labels = [ i for i in images_to_show if i not in all_labels ] random.shuffle(images_not_in_all_labels) random.shuffle(images_in_all_labels) images_to_show = images_not_in_all_labels + images_in_all_labels
from os import listdir from autowebcompat import utils labels_directory = 'label_persons/' all_file_names = [f for f in listdir(labels_directory)] labels_voted = {} ydn_map = {'y': 0, 'd': 1, 'n': 2} ydn_reverse_map = {0: 'y', 1: 'd', 2: 'n'} for file_name in all_file_names: labels = utils.read_labels(labels_directory + file_name) for key, value in labels.items(): if key not in labels_voted.keys(): labels_voted[key] = [0, 0, 0] labels_voted[key][ydn_map[value]] += 1 labels = {} for key, values in labels_voted.items(): labels[key] = ydn_reverse_map[values.index(max(values))] utils.write_labels(labels)
class Timer(Callback): def on_train_begin(self, logs={}): self.train_begin_time = time.time() self.epoch_times = [] def on_epoch_begin(self, batch, logs={}): self.epoch_begin_time = time.time() def on_epoch_end(self, batch, logs={}): self.epoch_times.append(time.time() - self.epoch_begin_time) def on_train_end(self, logs={}): self.train_time = time.time() - self.train_begin_time labels = utils.read_labels(args.labels) utils.prepare_images() all_image_names = [i for i in utils.get_images() if i in labels] all_images = sum([[i + '_firefox.png', i + '_chrome.png'] for i in all_image_names], []) image = utils.load_image(all_images[0]) input_shape = image.shape SAMPLE_SIZE = len(all_image_names) TRAIN_SAMPLE = 80 * (SAMPLE_SIZE // 100) VALIDATION_SAMPLE = 10 * (SAMPLE_SIZE // 100) TEST_SAMPLE = SAMPLE_SIZE - (TRAIN_SAMPLE + VALIDATION_SAMPLE) def load_pair(fname):
def test_read_labels(): labels = utils.read_labels(file_name='labels.csv') assert (isinstance(labels, dict))
def migrate_v1_to_v2(): def read_sequence(bug_id): with open('data/%d.txt' % bug_id) as f: return f.readlines() def write_sequence(bug_id, data): with open('./data/%d.txt' % bug_id, 'w') as f: for i in range(len(data)): for j in range(i + 1): f.write(data[j]) f.write('\n') all_data_files = os.listdir('data') label_files = os.listdir('label_persons') map_names = {} for i in range(7): map_names[i] = str(int((i + 1) * (i + 2) / 2 - 1)) for f in all_data_files: if '.png' not in f: continue parts = f.split('_') if len(parts) <= 2: continue bug_id = parts[0] seq_no = int(parts[1]) browser = parts[2] seq_no = map_names[seq_no] new_name = utils.create_file_name(bug_id=bug_id, browser=browser, seq_no=seq_no) os.rename(os.path.join('data', f), os.path.join('data', new_name)) for f in all_data_files: if '.txt' not in f: continue bug_id = os.path.splitext(f)[0] data = read_sequence(int(bug_id)) write_sequence(int(bug_id), data) for f in label_files: if 'csv' not in f: continue labels = utils.read_labels(os.path.join('label_persons', f)) new_labels = {} for key, value in labels.items(): key_info = utils.parse_file_name(key) if 'seq_no' not in key_info: new_labels[key] = value continue key = key.replace('_%d' % key_info['seq_no'], '_%s' % map_names[key_info['seq_no']]) new_labels[key] = value utils.write_labels(new_labels, os.path.join('label_persons', f)) for f in label_files: if 'json' not in f: continue bounding_boxes = utils.read_bounding_boxes(os.path.join('label_persons', f)) new_bounding_boxes = {} for key, value in bounding_boxes.items(): parts = key.split('_') if len(parts) <= 2: new_bounding_boxes[key] = value continue bug_id = parts[0] seq_no = int(parts[1]) browser = parts[2] seq_no = map_names[seq_no] key = utils.create_file_name(bug_id=bug_id, browser=browser, seq_no=seq_no) new_bounding_boxes[key] = value utils.write_bounding_boxes(new_bounding_boxes, os.path.join('label_persons', f))
def test_write_labels(tmpdir): label = {'1': '1', '2': '2'} file_path = tmpdir.join('test.csv') utils.write_labels(label, file_name=file_path) assert (os.path.exists(file_path)) assert (label == utils.read_labels(file_name=file_path))
def test_validate_labels(): labels = utils.read_labels() for label in labels.values(): assert label in ['y', 'n', 'd']