def getStatistics(conf):
n1 = 128
n2 = 0
#
# Change the transformations to only crop to the correct size.
t = transforms.Compose([
Perterbations.CenterCrop(conf.hyperparam.cropShape),
DataUtil.Rescale(conf.hyperparam.image_shape),
DataUtil.ToTensor(),
])
dataset = FlotDataset.FlotDataset(conf, conf.dataTrainList, t)
dataloader = torch.utils.data.DataLoader(dataset, batch_size = n1, num_workers = conf.numWorkers, shuffle = False, pin_memory = False)
#
# Iterate over the dataset and obtain individual means and std-devs.
# CURMEAN [ 0.18943557 0.21603064 0.21423657]
# curVAR [ 0.03896314 0.04170484 0.04159307]
totMean = 0
totVar = 0
numMini = len(dataloader)
pbar = tqdm.tqdm(total=numMini)
for data in dataloader:
imgs, labels_cpu = data['img'], data['labels']
curMean = np.mean(imgs.numpy(), axis=(0,2,3))
curVar = np.sqrt(np.var(imgs.numpy(), axis=(0,2,3)))
oldMean = totMean
totMean = (n1 * curMean + n2 * totMean) / (n1 + n2)
totVar = (n1 * curVar + n2 * totVar + n1 * np.power((curMean - totMean), 2) + n2 * np.power((oldMean - totMean), 2)) / (n1 + n2)
n2 += n1
pbar.update(1)
pbar.close()
#
# Print the final results.
print('Mean of the data: %s'%totMean)
print('stddev of the data: %s'%totVar)
def __init__(self, type='train'):
nSteps = (2, 0)
loadpath = '/Users/daniel/models/model_best.pth.tar'
if type == 'train':
loadpath = None
super(Config, self).__init__(MultiTraj_conv.Resnet_MultiConv(nSteps),
loadPath=loadpath)
#
# How far to shift the image.
self.hyperparam.shiftBounds = (90, 0)
self.hyperparam.cropShape = (448, 448)
self.hyperparam.nSteps = nSteps
self.hyperparam.numEpochs = 32
self.epochSaveInterval = 1
self.modelSavePath = '/disk1/model/'
#
# Transforms.
self.transforms = transforms.Compose([
Perterbations.RandomShift(self.hyperparam.cropShape,
self.hyperparam.shiftBounds,
self.hyperparam.nSteps, mode),
Perterbations.RandomHorizontalFlip(0.5, mode),
DataUtil.Rescale(self.hyperparam.image_shape),
Perterbations.ColourJitter(
0.7, 0.7, 0.7, 0.5,
mode), # The effects of this must be tuned.
DataUtil.ToTensor(),
])
self.dataTrainList = ['/Users/daniel/data/20180123_205116/']
def __init__(self, mode='train'):
nSteps = (2, 0)
loadpath = '/disk1/model/rl_multiconv-400x400-newdata-normalized_model_best.pth.tar'
if mode == 'train':
loadpath = None
super(Config, self).__init__(MultiTraj_conv.Resnet_MultiConv(nSteps),
loadPath=loadpath)
# super(Config, self).__init__(MultiTraj_FC.Resnet_Multifc(nSteps), loadPath = loadpath)
#
# The distance threshold for postive / negative.
self.distThreshold = 0.7
#
# How far to shift the image.
self.hyperparam.shiftBounds = (110, 0)
self.hyperparam.nSteps = nSteps
self.hyperparam.numEpochs = 32
self.hyperparam.cropShape = (400, 400)
self.epochSaveInterval = 1
self.modelSavePath = '/disk1/model/'
#
# Transforms.
mean = [0.31102816, 0.30806204, 0.290305]
std = [0.19752153, 0.19664317, 0.20129602]
self.denormalize = DataUtil.UnNormalize(mean, std)
self.normalize = DataUtil.Normalize(mean, std)
self.transforms = transforms.Compose([
Perterbations.RandomShift(self.hyperparam.cropShape,
self.hyperparam.shiftBounds,
self.hyperparam.nSteps, mode),
Perterbations.RandomHorizontalFlip(0.5, mode),
DataUtil.Rescale(self.hyperparam.image_shape),
Perterbations.ColourJitter(
0.05, 0.05, 0.05, 0.05,
mode), # The effects of this must be tuned.
DataUtil.ToTensor(),
self.normalize,
])
self.experimentName = 'rl_multiconv-400x400-newdata-normalized'
self.dataValList = [
'/disk1/rldata/20180304_042341',
'/disk1/rldata/20180306_012910',
]
self.dataTrainList = [
'/disk1/rldata/20180310_215333',
'/disk1/rldata/20180313_191316',
'/disk1/rldata/20180314_001703',
'/disk1/rldata/20180314_155329',
'/disk1/rldata/20180314_172112',
'/disk1/rldata/20180315_020652',
]
def __init__(self, mode = 'train'):
nSteps = (0, 0)
loadpath = '/disk1/model/rl_normalized-randomshift-lesscolour_model_best.pth.tar'
# loadpath = '/disk1/model/rl_newDataFixedLabel_model_best.pth.tar'
if mode == 'train':
loadpath = None
super(Config, self).__init__(model = GenericModel.GenericModel(models.resnet18(pretrained=True)), loadPath = loadpath)
#
# The distance threshold for postive / negative.
self.distThreshold = 0.7
#
# How far to shift the image.
self.hyperparam.shiftBounds = (30, 0)
self.hyperparam.nSteps = nSteps
self.hyperparam.numEpochs = 32
self.hyperparam.cropShape = (448, 448)
self.epochSaveInterval = 1
self.modelSavePath = '/disk1/model/'
#
# Transforms.
mean = [ 0.31102816, 0.30806204, 0.290305]
std = [0.19752153, 0.19664317, 0.20129602]
self.denormalize = DataUtil.UnNormalize(mean, std)
self.normalize = DataUtil.Normalize(mean, std)
self.transforms = transforms.Compose([
Perterbations.RandomShift(self.hyperparam.cropShape, self.hyperparam.shiftBounds, self.hyperparam.nSteps, mode),
# Perterbations.CenterCrop(self.hyperparam.cropShape),
DataUtil.Rescale(self.hyperparam.image_shape),
# Perterbations.CenterCrop(self.hyperparam.image_shape),
Perterbations.RandomHorizontalFlip(0.5, mode),
Perterbations.ColourJitter(0.05, 0.05, 0.05, 0.05, mode), # The effects of this must be tuned.
DataUtil.ToTensor(),
self.normalize,
])
self.experimentName = 'rl_normalized-fixeddata'
# self.dataValList = ['/disk1/rldata/20180306_012910',]
self.dataValList = [
'/disk1/rldata/20180304_042341',
'/disk1/rldata/20180306_012910',
]
self.dataTrainList = [
'/disk1/rldata/20180310_215333',
'/disk1/rldata/20180313_191316',
'/disk1/rldata/20180314_001703',
'/disk1/rldata/20180314_155329',
'/disk1/rldata/20180314_172112',
'/disk1/rldata/20180315_020652',
'/disk1/rldata/20180317_030020',
]
args = parser.parse_args()
return args
#
# Get the configuration, override as needed.
def getConfig(args):
config_module = __import__('config.' + args.configStr)
configuration = getattr(config_module, args.configStr)
conf = configuration.Config()
#
# Modifications to the configuration happen here.
return conf
#
# Main.
if __name__ == '__main__':
#
# The default configuration.
conf = getConfig(getInputArgs())
train = FlotDataset.FlotDataset(conf, conf.dataTrainList, conf.transforms)
dataset = torch.utils.data.DataLoader(train, batch_size = 32, num_workers = 1,
shuffle = True, pin_memory = False)
writer = SummaryWriter()
for data in dataset:
if torch.cuda.is_available():
writer.add_image('Image', data['img'].cuda(), 0)
else:
writer.add_image('Image', data['img'], 0)
writer.add_text('Text', 'text logged at step:'+str(1), 1)
DataUtil.plotSample(data)
break
writer.close()
def __init__(self, mode='train'):
nSteps = (0, 0)
loadpath = '/disk1/model/rl_normalized-randomshift-lesscolour_model_best.pth.tar'
# loadpath = '/disk1/model/rl_newDataFixedLabel_model_best.pth.tar'
if mode == 'train':
loadpath = None
super(Config, self).__init__(model=GenericModel.GenericModel(
models.resnet18(pretrained=True)),
loadPath=loadpath)
#
# The distance threshold for postive / negative.
self.distThreshold = 0.7
#
# How far to shift the image.
self.hyperparam.shiftBounds = (30, 0)
self.hyperparam.nSteps = nSteps
self.hyperparam.numEpochs = 32
self.hyperparam.cropShape = (448, 448)
self.epochSaveInterval = 1
self.modelSavePath = '/disk1/model/'
#
# Transforms.
mean = [0.31102816, 0.30806204, 0.290305]
std = [0.19752153, 0.19664317, 0.20129602]
self.denormalize = DataUtil.UnNormalize(mean, std)
self.normalize = DataUtil.Normalize(mean, std)
self.transforms = transforms.Compose([
Perterbations.RandomShift(self.hyperparam.cropShape,
self.hyperparam.shiftBounds,
self.hyperparam.nSteps, mode),
# Perterbations.CenterCrop(self.hyperparam.cropShape),
DataUtil.Rescale(self.hyperparam.image_shape),
# Perterbations.CenterCrop(self.hyperparam.image_shape),
Perterbations.RandomHorizontalFlip(0.5, mode),
Perterbations.ColourJitter(
0.05, 0.05, 0.05, 0.05,
mode), # The effects of this must be tuned.
DataUtil.ToTensor(),
self.normalize,
])
self.experimentName = 'mixed-fixedData'
# self.dataValList = ['/disk1/rldata/20180306_012910',]
self.dataValList = [
'/disk1/rldata/20180304_042341',
'/disk1/rldata/20180306_012910',
# SIM
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-02-31-14/',
'/disk1/data/20171202-003453/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-01-15-47/',
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-03-22-14/',
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-03-32-26/',
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-03-42-38/',
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-01-40-14/',
]
self.dataTrainList = [
'/disk1/rldata/20180310_215333',
'/disk1/rldata/20180313_191316',
'/disk1/rldata/20180314_001703',
'/disk1/rldata/20180314_155329',
'/disk1/rldata/20180314_172112',
'/disk1/rldata/20180315_020652',
'/disk1/rldata/20180317_030020',
#
# SIM
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-02-00-38/',
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-02-41-26/',
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-01-50-26/',
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-01-30-02/',
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-02-21-02/',
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-02-10-50/',
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-03-01-50/',
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-01-19-50/',
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-02-51-38/',
'/disk1/data/20171202-011944/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-03-12-02/',
'/disk1/data/20171202-003453/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-00-34-59/',
'/disk1/data/20171202-003453/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-00-55-23/',
'/disk1/data/20171202-003453/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-01-05-35/',
'/disk1/data/20171202-003453/dumbAgent_EnvironmentTypes.AirSim_02-12-2017-00-45-11/',
#
# # Dagger 1.
'/disk1/data/dagger1/20171202-115839/dataCollectionAgent_EnvironmentTypes.AirSim_02-12-2017-11-58-54/',
'/disk1/data/dagger1/20171202-115410/dataCollectionAgent_EnvironmentTypes.AirSim_02-12-2017-11-54-25/',
'/disk1/data/dagger1/20171202-113449/dataCollectionAgent_EnvironmentTypes.AirSim_02-12-2017-11-35-04/',
'/disk1/data/dagger1/20171202-133243/dataCollectionAgent_EnvironmentTypes.AirSim_02-12-2017-13-32-59/',
# #
# # Dagger 2.
'/disk1/data/dagger2/20171202-140740/dataCollectionAgent_EnvironmentTypes.AirSim_02-12-2017-14-07-55/',
'/disk1/data/dagger2/20171202-141434/dataCollectionAgent_EnvironmentTypes.AirSim_02-12-2017-14-14-50/',
'/disk1/data/dagger2/20171202-141434/dataCollectionAgent_EnvironmentTypes.AirSim_02-12-2017-14-35-24/',
'/disk1/data/dagger2/20171202-141434/dataCollectionAgent_EnvironmentTypes.AirSim_02-12-2017-14-25-06/',
'/disk1/data/dagger2/20171202-141434/dataCollectionAgent_EnvironmentTypes.AirSim_02-12-2017-14-55-58/',
'/disk1/data/dagger2/20171202-135535/dataCollectionAgent_EnvironmentTypes.AirSim_02-12-2017-13-55-50/',
'/disk1/data/dagger2/20171202-141434/dataCollectionAgent_EnvironmentTypes.AirSim_02-12-2017-14-45-41/',
]