def testPut(self):
# create domain
domain = 'valueput.datasettest.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # creates domain
#create 1d dataset
payload = {'type': 'H5T_STD_I32LE', 'shape': 10}
req = self.endpoint + "/datasets/"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset1UUID = rspJson['id']
self.assertTrue(helper.validateId(dset1UUID))
# link new dataset as 'dset1'
ok = helper.linkObject(domain, dset1UUID, 'dset1')
self.assertTrue(ok)
req = self.endpoint + "/datasets/" + dset1UUID + "/value"
data = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29]
payload = {'type': 'H5T_STD_I32LE', 'shape': 10, 'value': data}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# read back the data
readData = helper.readDataset(domain, dset1UUID)
self.failUnlessEqual(readData,
data) # verify we got back what we started with
#create 2d dataset
payload = {'type': 'H5T_STD_I32LE', 'shape': (10, 10)}
req = self.endpoint + "/datasets/"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset2UUID = rspJson['id']
self.assertTrue(helper.validateId(dset2UUID))
# link new dataset as 'dset2'
ok = helper.linkObject(domain, dset2UUID, 'dset2')
self.assertTrue(ok)
req = self.endpoint + "/datasets/" + dset2UUID + "/value"
data = []
for i in range(10):
row = []
for j in range(10):
row.append(i * 10 + j)
data.append(row)
payload = {'type': 'H5T_STD_I32LE', 'shape': [10, 10], 'value': data}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# read back the data
readData = helper.readDataset(domain, dset2UUID)
self.failUnlessEqual(readData,
data) # verify we got back what we started with
def testPut(self):
# create domain
domain = 'valueput.datasettest.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # creates domain
#create 1d dataset
payload = {'type': 'H5T_STD_I32LE', 'shape': 10}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset1UUID = rspJson['id']
self.assertTrue(helper.validateId(dset1UUID))
# link new dataset as 'dset1'
ok = helper.linkObject(domain, dset1UUID, 'dset1')
self.assertTrue(ok)
req = self.endpoint + "/datasets/" + dset1UUID + "/value"
data = [2,3,5,7,11,13,17,19,23,29]
# payload = {'type': 'H5T_STD_I32LE', 'shape': 10, 'value': data }
payload = { 'value': data }
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# read back the data
readData = helper.readDataset(domain, dset1UUID)
self.failUnlessEqual(readData, data) # verify we got back what we started with
#create 2d dataset
payload = {'type': 'H5T_STD_I32LE', 'shape': (10,10)}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset2UUID = rspJson['id']
self.assertTrue(helper.validateId(dset2UUID))
# link new dataset as 'dset2'
ok = helper.linkObject(domain, dset2UUID, 'dset2')
self.assertTrue(ok)
req = self.endpoint + "/datasets/" + dset2UUID + "/value"
data = []
for i in range(10):
row = []
for j in range(10):
row.append(i*10 + j)
data.append(row)
payload = { 'value': data }
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# read back the data
readData = helper.readDataset(domain, dset2UUID)
self.failUnlessEqual(readData, data) # verify we got back what we started with
def testPutPointSelection(self):
# create domain
domain = 'valueputpointsel.datasettest.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # creates domain
#create 1d dataset
payload = {'type': 'H5T_STD_I32LE', 'shape': 100}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset1UUID = rspJson['id']
self.assertTrue(helper.validateId(dset1UUID))
# link new dataset as 'dset1'
ok = helper.linkObject(domain, dset1UUID, 'dset1')
self.assertTrue(ok)
req = self.endpoint + "/datasets/" + dset1UUID + "/value"
primes = [2,3,5,7,11,13,17,19,23,29,31,37,41,43,47,53,59,61,67,71,73,79,83,89,97]
value = [1,] * len(primes) # write 1's at indexes that are prime
# write 1's to all the prime indexes
payload = { 'points': primes, 'value': value }
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# read back the data
readData = helper.readDataset(domain, dset1UUID)
self.failUnlessEqual(readData[37], 1) # prime
self.failUnlessEqual(readData[38], 0) # not prime
def testPutSelection(self):
# create domain
domain = 'valueputsel.datasettest.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # creates domain
#create 1d dataset
payload = {'type': 'H5T_STD_I32LE', 'shape': 10}
req = self.endpoint + "/datasets/"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset1UUID = rspJson['id']
self.assertTrue(helper.validateId(dset1UUID))
# link new dataset as 'dset1'
ok = helper.linkObject(domain, dset1UUID, 'dset1')
self.assertTrue(ok)
req = self.endpoint + "/datasets/" + dset1UUID + "/value"
data = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29]
data_part1 = data[0:5]
data_part2 = data[5:10]
# write part 1
payload = {
'type': 'H5T_STD_I32LE',
'shape': 5,
'start': 0,
'stop': 5,
'value': data_part1
}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# write part 2
payload = {
'type': 'H5T_STD_I32LE',
'shape': 5,
'start': 5,
'stop': 10,
'value': data_part2
}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# read back the data
readData = helper.readDataset(domain, dset1UUID)
self.failUnlessEqual(readData,
data) # verify we got back what we started with
def main():
random.seed(1000)
image_filenames = []
angles = []
image_filenames, angles = helper.readDataset(image_filenames, angles,
'./data/driving_log.csv',
'./data/IMG/')
#image_filenames, angles = helper.readDataset(image_filenames, angles, './recorded/driving_log.csv','./recorded/IMG/')
image_filenames, angles = helper.removeOverrepresentedData(
image_filenames, angles)
if visualization_mode == 1:
helper.visualizeDataDistribution(angles)
image_filenames_train, image_filenames_test, angles_train, angles_test = train_test_split(
image_filenames, angles, test_size=0.05, random_state=42)
# compile and train the model using the generator function
train_generator = helper.samples_generator(image_filenames_train,
angles_train,
batch_size=32)
validation_generator = helper.samples_generator(image_filenames_test,
angles_test,
batch_size=32)
if visualization_mode == 1:
while 1:
X, y = next(train_generator)
helper.visualizeDataset(X, y)
else:
checkpoint = ModelCheckpoint('model{epoch:02d}.h5')
model = helper.build_nvidia_model()
model.compile(optimizer=Adam(lr=1e-4), loss='mse')
model.fit_generator(train_generator,
samples_per_epoch=len(angles_train),
validation_data=validation_generator,
nb_val_samples=len(angles_test),
nb_epoch=5,
callbacks=[checkpoint])
model.save('model.h5')
def testPutSelection(self):
# create domain
domain = 'valueputsel.datasettest.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # creates domain
#create 1d dataset
payload = {'type': 'H5T_STD_I32LE', 'shape': 10}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset1UUID = rspJson['id']
self.assertTrue(helper.validateId(dset1UUID))
# link new dataset as 'dset1'
ok = helper.linkObject(domain, dset1UUID, 'dset1')
self.assertTrue(ok)
req = self.endpoint + "/datasets/" + dset1UUID + "/value"
data = [2,3,5,7,11,13,17,19,23,29]
data_part1 = data[0:5]
data_part2 = data[5:10]
# write part 1
payload = { 'start': 0, 'stop': 5, 'value': data_part1 }
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# write part 2
payload = { 'start': 5, 'stop': 10, 'value': data_part2 }
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# read back the data
readData = helper.readDataset(domain, dset1UUID)
self.failUnlessEqual(readData, data) # verify we got back what we started with
def testPutPointSelection(self):
# create domain
domain = 'valueputpointsel.datasettest.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # creates domain
#create 1d dataset
payload = {'type': 'H5T_STD_I32LE', 'shape': 100}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset1UUID = rspJson['id']
self.assertTrue(helper.validateId(dset1UUID))
# link new dataset as 'dset1'
ok = helper.linkObject(domain, dset1UUID, 'dset1')
self.assertTrue(ok)
req = self.endpoint + "/datasets/" + dset1UUID + "/value"
primes = [
2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61,
67, 71, 73, 79, 83, 89, 97
]
value = [
1,
] * len(primes) # write 1's at indexes that are prime
# write 1's to all the prime indexes
payload = {'points': primes, 'value': value}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# read back the data
readData = helper.readDataset(domain, dset1UUID)
self.failUnlessEqual(readData[37], 1) # prime
self.failUnlessEqual(readData[38], 0) # not prime