def chips_and_answers():
MAKES = ("Honda", None,)
MODELS = ("Civic", None,)
COLORS = ("Red", None,)
answer = namedtuple("answer", ["make", "model", "color"])
chips_and_answers = []
for make, model, color in product(MAKES, MODELS, COLORS):
chip = Chip(None, None, None, None, {"make": make, "model": model, "color": color, "other_key": "DO NOT SELECT THIS"})
ans = answer(make, model, color)
chips_and_answers.append((chip, ans))
# If misc is missing, it should also work
chip = Chip(None, None, None, None, None)
ans = answer(None, None, None)
chips_and_answers.append((chip, ans))
# If keys are missing, it should also work
chip = Chip(None, None, None, None, {"other_key": "DO NOT SELECT THIS"})
ans = answer(None, None, None)
chips_and_answers.append((chip, ans))
# If misc is missing, it should still work
fake_chip = namedtuple("fakechip", ["not_misc"])
chip = fake_chip(None)
ans = answer(None, None, None)
chips_and_answers.append((chip, ans))
return chips_and_answers
def make_model_color_classes():
MAKES = ("Honda", "Toyota", None,)
MODELS = ("Civic", "Corolla", None,)
COLORS = ("Red", "Blue", None,)
# Make all possible sets of three "chips", including with replacement, so
# for example we'll get:
#
# (Honda, Honda, Honda)
# (Honda, Honda, Toyota)
# (Honda, Toyota, None)
#
# And others
NUMBER_OF_CHIPS = 3
MAKE_P = combinations_with_replacement(MAKES, NUMBER_OF_CHIPS)
MODEL_P = combinations_with_replacement(MODELS, NUMBER_OF_CHIPS)
COLOR_P = combinations_with_replacement(COLORS, NUMBER_OF_CHIPS)
answer = namedtuple("answer", ["make", "model", "color"])
chips_and_answers = []
for makes, models, colors in product(MAKE_P, MODEL_P, COLOR_P):
chips = []
ans = []
for make, model, color in zip(makes, models, colors):
chips.append(Chip(None, None, None, None, {"make": make, "model": model, "color": color, "other_key": "DO NOT SELECT THIS"}))
ans.append(answer(make, model, color))
chips_and_answers.append((tuple(chips), tuple(ans)))
return chips_and_answers
def load(filename):
with h5py.File(filename) as fIn:
feats = np.array(fIn['feats'])
num_items = fIn['feats'].shape[0]
# Hack to deal with performance of extracting single items
local_hdf5 = {}
local_hdf5['chip_keys'] = np.array(fIn['chip_keys'])
local_hdf5['filepath'] = np.array(fIn['filepath'])
local_hdf5['car_id'] = np.array(fIn['car_id'])
local_hdf5['cam_id'] = np.array(fIn['cam_id'])
local_hdf5['time'] = np.array(fIn['time'])
local_hdf5['misc'] = np.array(fIn['misc'])
chips = {}
chip_index_lookup = {}
for i in range(num_items):
filepath = local_hdf5['filepath'][i].decode('utf-8')
car_id = local_hdf5['car_id'][i]
cam_id = local_hdf5['cam_id'][i]
timestamp = local_hdf5['time'][i]
if isinstance(timestamp, str) or isinstance(timestamp, bytes):
# Catch the case where we have encoded time as a string timestamp
timestamp = datetime.datetime.fromtimestamp(
float(timestamp))
misc = json.loads(local_hdf5['misc'][i].decode('utf-8'))
chip_key = local_hdf5['chip_keys'][i]
if isinstance(chip_key, bytes):
chip_key = chip_key.decode('utf-8')
chip_index_lookup[chip_key] = i
chips[chip_key] = Chip(filepath, car_id, cam_id, timestamp,
misc)
return chip_index_lookup, chips, feats
def dgcars(tmpdir):
# Define some test and training data, all will be the sum
TRAIN = [
{"url": "http://example.com/img.jpg", "hash": "2a8cedfa145b4345aed3fd9e82796c3e", "resnet50": "minivan", "model": "ZX2", "filename": "black/Ford/2a8cedfa145b4345aed3fd9e82796c3e.jpg", "make": "Ford", "color": "black"},
{"url": "http://example.com/img.jpg", "hash": "8241daf452ace679162c69386f26ddc7", "resnet50": "sports_car", "model": "Mazda6 Sport", "filename": "red/Mazda/8241daf452ace679162c69386f26ddc7.jpg", "make": "Mazda", "color": "red"},
{"url": "http://example.com/img.jpg", "hash": "e8dc3fb78206b14fe3568c1b28e5e5a1", "resnet50": "cab", "model": "XJ Series", "filename": "yellow/Jaguar/e8dc3fb78206b14fe3568c1b28e5e5a1.jpg", "make": "Jaguar", "color": "yellow"},
]
TEST = [
{"url": "http://example.com/img.jpg", "hash": "8881e7b561393f1d778a70dd449433e9", "resnet50": "racer", "model": "IS F", "filename": "yellow/Lexus/8881e7b561393f1d778a70dd449433e9.jpg", "make": "Lexus", "color": "yellow"},
{"url": "http://example.com/img.jpg", "hash": "38e857d5235afda4315676c0b7756832", "resnet50": "pickup", "model": "Mark VII", "filename": "silver/Lincoln/38e857d5235afda4315676c0b7756832.jpg", "make": "Lincoln", "color": "silver"},
{"url": "http://example.com/img.jpg", "hash": "6eb2b407cc398e70604bfd336bb2efad", "resnet50": "pickup", "model": "Lightning", "filename": "orange/Ford/6eb2b407cc398e70604bfd336bb2efad.jpg", "make": "Ford", "color": "orange"},
{"url": "http://example.com/img.jpg", "hash": "eb3811772ec012545c8952d88906d355", "resnet50": "racer", "model": "Rockette", "filename": "green/Fairthorpe/eb3811772ec012545c8952d88906d355.jpg", "make": "Fairthorpe", "color": "green"},
{"url": "http://example.com/img.jpg", "hash": "8dbbc1d930c7f2e4558efcc596728945", "resnet50": "minivan", "model": "S70", "filename": "white/Volvo/8dbbc1d930c7f2e4558efcc596728945.jpg", "make": "Volvo", "color": "white"},
{"url": "http://example.com/img.jpg", "hash": "ed45784812d1281bcb61f217f4422ab5", "resnet50": "convertible", "model": "A8", "filename": "green/Audi/ed45784812d1281bcb61f217f4422ab5.jpg", "make": "Audi", "color": "green"},
{"url": "http://example.com/img.jpg", "hash": "763ca4abbbb9b042b21f19fd80986179", "resnet50": "pickup", "model": "W126", "filename": "green/Mercedes-Benz/763ca4abbbb9b042b21f19fd80986179.jpg", "make": "Mercedes-Benz", "color": "green"},
]
WRITE_LIST = (
# filename, data list, settype
("allFiles", TRAIN + TEST, SetType.ALL),
("training", TRAIN, SetType.TRAIN),
("testing", TEST, SetType.TEST),
)
output_chips = {
SetType.ALL: [],
SetType.TRAIN: [],
SetType.TEST: [],
}
for filename, data_list, settype in WRITE_LIST:
fn = tmpdir.join(filename)
with open(fn.strpath, "w") as f:
for d in data_list:
# Write the data list files
line = json.dumps(d)
f.write(line + "\n")
# Make a chip
fp = os.path.join(tmpdir.strpath, d["filename"])
chip = Chip(fp, None, None, None, d)
output_chips[settype].append(chip)
# Instantiate a DGCarsDataset() class
output_classes = {
SetType.ALL: DGCarsDataset(tmpdir.strpath, SetType.ALL.value),
SetType.TRAIN: DGCarsDataset(tmpdir.strpath, SetType.TRAIN.value),
SetType.TEST: DGCarsDataset(tmpdir.strpath, SetType.TEST.value),
}
return (output_classes, output_chips)
def chips():
CHIPS = (
# filepath, car_id, cam_id, time, misc
("car1_cam1.png", 1, 1, datetime.datetime(2016, 10,1, 0, 1, 2, microsecond=100), {}),
("car1_cam2.png", 1, 2, datetime.datetime(2016, 10,1, 0, 1, 2, microsecond=105), {}),
("car1_cam3.png", 1, 3, datetime.datetime(2016, 10,1, 0, 1, 2, microsecond=110), {}),
("car2_cam1.png", 2, 1, datetime.datetime(2016, 10,1, 0, 1, 2, microsecond=100), {}),
("car2_cam2.png", 2, 1, datetime.datetime(2016, 10,1, 0, 1, 2, microsecond=102), {}),
("car2_cam3.png", 2, 1, datetime.datetime(2016, 10,1, 0, 1, 2, microsecond=104), {}),
)
chips = {}
for filepath, car_id, cam_id, time, misc in CHIPS:
chip = Chip(filepath, car_id, cam_id, time, misc)
chips[filepath] = chip
return chips
def fake_dataset(tmpdir):
MAKE_MODELS = (
("honda", "civic"),
("toyota", "corolla"),
)
# File names must start at 0 and increment by 1. They must be assigned to
# the tuples after the tuples have been sorted alphabetically.
chips = {}
for i, (make, model) in enumerate(sorted(MAKE_MODELS)):
name = str(i) + ".jpg"
fn = tmpdir.join(name)
fn.write("FAKE IMAGE")
chip = Chip(str(fn), None, None, None, {"make": make, "model": model})
chips[i] = chip
return FakeChipDataset(chips, None)
def chips():
CHIPS = (
# filepath, car_id, cam_id, time, misc
("car1_cam1.png", 1, 1, 100, None),
("car1_cam2.png", 1, 2, 105, None),
("car1_cam3.png", 1, 3, 110, None),
("car2_cam1.png", 2, 1, 100, None),
("car2_cam2.png", 2, 1, 102, None),
("car2_cam3.png", 2, 1, 104, None),
)
chips = {}
for filepath, car_id, cam_id, time, misc in CHIPS:
chip = Chip(filepath, car_id, cam_id, time, misc)
chips[filepath] = chip
return chips