def val(spatial_size, Scale, precomputeStride):
d = pickle.load(open('pickle/test.pickle', 'rb'))
d = torchnet.dataset.ListDataset(d)
randperm = torch.randperm(len(d))
def perm(idx, size):
return randperm[idx]
def merge(tbl):
inp = scn.InputBatch(2, spatial_size)
center = spatial_size.float().view(1, 2) / 2
p = torch.LongTensor(2)
v = torch.FloatTensor([1, 0, 0])
for char in tbl['input']:
inp.addSample()
for stroke in char:
stroke = stroke.float() * (Scale - 0.01) / 255 - 0.5 * (Scale - 0.01)
stroke += center.expand_as(stroke)
scn.dim_fn(
2,
'drawCurve')(
inp.metadata.ffi,
inp.features,
stroke)
inp.precomputeMetadata(precomputeStride)
return {'input': inp, 'target': torch.LongTensor(tbl['target']) - 1}
bd = torchnet.dataset.BatchDataset(d, 183, perm=perm, merge=merge)
tdi = scn.threadDatasetIterator(bd)
def iter():
randperm.copy_(torch.randperm(len(d)))
return tdi()
return iter
def val(spatial_size, Scale, precomputeStride):
d = pickle.load(open('pickle/test.pickle', 'rb'))
d = torchnet.dataset.ListDataset(d)
randperm = torch.randperm(len(d))
def perm(idx, size):
return randperm[idx]
def merge(tbl):
inp = scn.InputBatch(2, spatial_size)
center = spatial_size.float().view(1, 2) / 2
p = torch.LongTensor(2)
v = torch.FloatTensor([1, 0, 0])
for char in tbl['input']:
inp.addSample()
for stroke in char:
stroke = stroke.float() * (Scale - 0.01) / 255 - 0.5 * (Scale - 0.01)
stroke += center.expand_as(stroke)
scn.dim_fn(
2,
'drawCurve')(
inp.metadata.ffi,
inp.features,
stroke)
inp.precomputeMetadata(precomputeStride)
return {'input': inp, 'target': torch.LongTensor(tbl['target']) - 1}
bd = torchnet.dataset.BatchDataset(d, 183, perm=perm, merge=merge)
tdi = scn.threadDatasetIterator(bd)
def iter():
randperm = torch.randperm(len(d))
return tdi()
return iter
def train(spatial_size, Scale, precomputeSize):
d = pickle.load(open('pickle/train.pickle', 'rb'))
d = torchnet.dataset.ListDataset(d)
randperm = torch.randperm(len(d))
def perm(idx, size):
return randperm[idx]
def merge(tbl):
inp = scn.InputBatch(2, spatial_size)
center = spatial_size.float().view(1, 2) / 2
p = torch.LongTensor(2)
v = torch.FloatTensor([1, 0, 0])
for char in tbl['input']:
inp.add_sample()
for stroke in char:
stroke = stroke.float() * (Scale - 0.01) / 255 - 0.5 * (Scale - 0.01)
stroke += center.expand_as(stroke)
###############################################################
# To avoid GIL problems use a helper function:
scn.dim_fn(
2,
'drawCurve')(
inp.metadata.ffi,
inp.features,
stroke)
###############################################################
# Above is equivalent to :
# x1,x2,y1,y2,l=0,stroke[0][0],0,stroke[0][1],0
# for i in range(1,stroke.size(0)):
# x1=x2
# y1=y2
# x2=stroke[i][0]
# y2=stroke[i][1]
# l=1e-10+((x2-x1)**2+(y2-y1)**2)**0.5
# v[1]=(x2-x1)/l
# v[2]=(y2-y1)/l
# l=max(x2-x1,y2-y1,x1-x2,y1-y2,0.9)
# for j in numpy.arange(0,1,1/l):
# p[0]=math.floor(x1*j+x2*(1-j))
# p[1]=math.floor(y1*j+y2*(1-j))
# inp.set_location(p,v,False)
###############################################################
inp.precomputeMetadata(precomputeSize)
return {'input': inp, 'target': torch.LongTensor(tbl['target'])}
bd = torchnet.dataset.BatchDataset(d, 100, perm=perm, merge=merge)
tdi = scn.threadDatasetIterator(bd)
def iter():
randperm.copy_(torch.randperm(len(d)))
return tdi()
return iter
def train(spatial_size, Scale, precomputeStride):
d = pickle.load(open('pickle/train.pickle', 'rb'))
print('Replicating training set 10 times (1 epoch = 10 iterations through the training set = 10x6588 training samples)')
for i in range(9):
for j in range(6588):
d.append(d[j])
d = torchnet.dataset.ListDataset(d)
randperm = torch.randperm(len(d))
def perm(idx, size):
return randperm[idx]
def merge(tbl):
inp = scn.InputBatch(2, spatial_size)
center = spatial_size.float().view(1, 2) / 2
p = torch.LongTensor(2)
v = torch.FloatTensor([1, 0, 0])
for char in tbl['input']:
inp.addSample()
m = torch.eye(2)
r = random.randint(1, 3)
alpha = random.uniform(-0.2, 0.2)
if alpha == 1:
m[0][1] = alpha
elif alpha == 2:
m[1][0] = alpha
else:
m = torch.mm(m, torch.FloatTensor(
[[math.cos(alpha), math.sin(alpha)],
[-math.sin(alpha), math.cos(alpha)]]))
c = center + torch.FloatTensor(1, 2).uniform_(-8, 8)
for stroke in char:
stroke = stroke.float() / 255 - 0.5
stroke = c.expand_as(stroke) + \
torch.mm(stroke, m * (Scale - 0.01))
###############################################################
# To avoid GIL problems use a helper function:
scn.dim_fn(
2,
'drawCurve')(
inp.metadata.ffi,
inp.features,
stroke)
###############################################################
# Above is equivalent to :
# x1,x2,y1,y2,l=0,stroke[0][0],0,stroke[0][1],0
# for i in range(1,stroke.size(0)):
# x1=x2
# y1=y2
# x2=stroke[i][0]
# y2=stroke[i][1]
# l=1e-10+((x2-x1)**2+(y2-y1)**2)**0.5
# v[1]=(x2-x1)/l
# v[2]=(y2-y1)/l
# l=max(x2-x1,y2-y1,x1-x2,y1-y2,0.9)
# for j in numpy.arange(0,1,1/l):
# p[0]=math.floor(x1*j+x2*(1-j))
# p[1]=math.floor(y1*j+y2*(1-j))
# inp.setLocation(p,v,False)
###############################################################
inp.precomputeMetadata(precomputeStride)
return {'input': inp, 'target': torch.LongTensor(tbl['target']) - 1}
bd = torchnet.dataset.BatchDataset(d, 108, perm=perm, merge=merge)
tdi = scn.threadDatasetIterator(bd)
def iter():
randperm.copy_(torch.randperm(len(d)))
return tdi()
return iter
def train(spatial_size, Scale, precomputeStride):
d = pickle.load(open('pickle/train.pickle', 'rb'))
print('Replicating training set 10 times (1 epoch = 10 iterations through the training set = 10x6588 training samples)')
for i in range(9):
for j in range(6588):
d.append(d[j])
d = torchnet.dataset.ListDataset(d)
randperm = torch.randperm(len(d))
def perm(idx, size):
return randperm[idx]
def merge(tbl):
inp = scn.InputBatch(2, spatial_size)
center = spatial_size.float().view(1, 2) / 2
p = torch.LongTensor(2)
v = torch.FloatTensor([1, 0, 0])
for char in tbl['input']:
inp.addSample()
m = torch.eye(2)
r = random.randint(1, 3)
alpha = random.uniform(-0.2, 0.2)
if alpha == 1:
m[0][1] = alpha
elif alpha == 2:
m[1][0] = alpha
else:
m = torch.mm(m, torch.FloatTensor(
[[math.cos(alpha), math.sin(alpha)],
[-math.sin(alpha), math.cos(alpha)]]))
c = center + torch.FloatTensor(1, 2).uniform_(-8, 8)
for stroke in char:
stroke = stroke.float() / 255 - 0.5
stroke = c.expand_as(stroke) + \
torch.mm(stroke, m * (Scale - 0.01))
###############################################################
# To avoid GIL problems use a helper function:
scn.dim_fn(
2,
'drawCurve')(
inp.metadata.ffi,
inp.features,
stroke)
###############################################################
# Above is equivalent to :
# x1,x2,y1,y2,l=0,stroke[0][0],0,stroke[0][1],0
# for i in range(1,stroke.size(0)):
# x1=x2
# y1=y2
# x2=stroke[i][0]
# y2=stroke[i][1]
# l=1e-10+((x2-x1)**2+(y2-y1)**2)**0.5
# v[1]=(x2-x1)/l
# v[2]=(y2-y1)/l
# l=max(x2-x1,y2-y1,x1-x2,y1-y2,0.9)
# for j in numpy.arange(0,1,1/l):
# p[0]=math.floor(x1*j+x2*(1-j))
# p[1]=math.floor(y1*j+y2*(1-j))
# inp.setLocation(p,v,False)
###############################################################
inp.precomputeMetadata(precomputeStride)
return {'input': inp, 'target': torch.LongTensor(tbl['target']) - 1}
bd = torchnet.dataset.BatchDataset(d, 108, perm=perm, merge=merge)
tdi = scn.threadDatasetIterator(bd)
def iter():
randperm = torch.randperm(len(d))
return tdi()
return iter
def train(DATASET, used_train_num):
spatialSize = DATASET['spatialSize']
output_spatialSize = DATASET['output_spatialSize']
dataset_outputSize = DATASET['dataset_outputSize']
dataset_outputSize2 = DATASET['dataset_outputSize2']
train_data_path = DATASET['train_data_path']
train_data_num = DATASET['train_data_num']
train_weight_path = DATASET['train_weight_path']
input_offset = DATASET['input_offset']
output_offset = DATASET['output_offset']
train_batch_size = DATASET['train_batch_size']
easy_ratio = DATASET['easy_ratio']
neg_ratio = DATASET['neg_ratio']
d = range(train_data_num)
def loadData(idx):
data = msgpack.load(open(train_data_path + str(idx) + '.msg', 'rb'))
while np.asarray(data[b'input_nz']).size == 0: # check
print('discard data:' + train_data_path + str(idx) + '.msg')
idx = np.random.randint(train_data_num)
data = msgpack.load(open(train_data_path + str(idx) + '.msg',
'rb'))
data[b'weight'] = msgpack.load(
open(train_weight_path + str(idx) + '_weight.msg', 'rb'))
return data
d = torchnet.dataset.ListDataset(d, load=loadData)
randperm = torch.randperm(len(d))
def sampler(dataset, idx):
return randperm[idx]
d = torchnet.dataset.ResampleDataset(d,
sampler=sampler,
size=used_train_num *
train_batch_size)
def perm(idx, size):
return idx
def merge(tbl):
merge_time = time.time()
input = scn.InputBatch(3, spatialSize)
target = []
target2 = []
batch_weight = []
batch_weight2 = []
count = 0
locations = np.empty((0, 3)).astype(int)
vals = []
nz_nums = torch.LongTensor(train_batch_size)
for input_nz, input_val, target_nz, target_val, weight_info in zip(
tbl[b'input_nz'], tbl[b'input_val'], tbl[b'target_nz'],
tbl[b'target_val'], tbl[b'weight']):
locations = np.vstack((locations, input_nz))
vals = np.concatenate([vals, input_val])
nz_nums[count] = len(input_nz)
label = np.zeros(dataset_outputSize.tolist()).astype(np.float32)
label[target_nz.astype(int).tolist()] = target_val
target.append(label)
# compute downscale label
label2 = SUNCGData.downsampleLabel2(label, 2).reshape(
dataset_outputSize2.tolist())
target2.append(label2)
# compute weight
weight = np.zeros(dataset_outputSize.tolist())
weight[weight_info[b'hard_pos'].astype(int).tolist()] = 1
hard_pos_num = weight_info[b'hard_pos'].shape[1]
easy_pos_num = min(int(hard_pos_num * easy_ratio),
weight_info[b'easy_pos'].shape[1])
if easy_pos_num > 0:
easy_pos_idx = np.random.permutation(
weight_info[b'easy_pos'].shape[1])[:easy_pos_num]
weight[weight_info[b'easy_pos'][:, easy_pos_idx].astype(
int).tolist()] = 1
neg_num = int(
min((hard_pos_num + easy_pos_num) * neg_ratio,
weight_info[b'hard_neg'].shape[1] / (1 - easy_ratio)))
hard_neg_num = int(
min(neg_num * (1 - easy_ratio),
weight_info[b'hard_neg'].shape[1]))
if hard_neg_num > 0:
hard_neg_idx = np.random.permutation(
weight_info[b'hard_neg'].shape[1])[:hard_neg_num]
weight[weight_info[b'hard_neg'][:, hard_neg_idx].astype(
int).tolist()] = 1
easy_neg_num = neg_num - hard_neg_num
if easy_neg_num > 0:
easy_neg_idx = np.random.permutation(
weight_info[b'easy_neg'].shape[1])[:easy_neg_num]
weight[weight_info[b'easy_neg'][:, easy_neg_idx].astype(
int).tolist()] = 1
batch_weight.append(weight)
weight2 = weight.reshape([
dataset_outputSize2[0], 2, dataset_outputSize2[1], 2,
dataset_outputSize2[2], 2
]).mean(5).mean(3).mean(1)
weight2[weight2 > 0] = 1
batch_weight2.append(weight2)
count = count + 1
#put data at the center of the volume
input.setInputBatchLocations(torch.from_numpy(locations).long()+input_offset.view(1,3), \
torch.FloatTensor(vals).view(nz_nums.sum(),1), nz_nums)
precomputeStride = 2
input.precomputeMetadata(precomputeStride)
return {'input': input, 'target': torch.LongTensor(np.array(target).astype(int)), 'weight':torch.FloatTensor(np.array(batch_weight)), \
'target2': torch.LongTensor(np.array(target2).astype(int)), 'weight2':torch.FloatTensor(np.array(batch_weight2)) }
bd = torchnet.dataset.BatchDataset(d,
train_batch_size,
perm=perm,
merge=merge)
tdi = scn.threadDatasetIterator(bd)
def iter():
return tdi()
return iter
