def readIMU_File(self, path):
# imu = []
# count = 0
# with open(self.base_dir + self.sequence + path) as csvfile:
# spamreader = csv.reader(csvfile, delimiter=',', quotechar='|')
# for row in spamreader:
# if count == 0:
# count += 1
# continue
# parsed = [float(row[1]), float(row[2]), float(row[3]),
# float(row[4]), float(row[5]), float(row[6])]
# imu.append(parsed)
#
# return np.array(imu)
imu = []
count = 0
with open(self.base_dir + self.sequence + path) as data:
for line in data.readlines():
row = line.strip().split(' ')
if count == 0:
count += 1
continue
parsed = [
float(row[1]),
float(row[2]),
float(row[3]),
float(row[4]),
float(row[5]),
float(row[6])
]
imu.append(parsed)
return np.array(imu)
def readTrajectoryFile(self, path):
# traj = []
# with open(self.base_dir + self.sequence + path) as csvfile:
# spamreader = csv.reader(csvfile, delimiter=',', quotechar='|')
# for row in spamreader:
# parsed = [float(row[0]), float(row[1]), float(row[2]), float(row[3]),
# float(row[4]), float(row[5]), float(row[6]), float(row[7])]
# traj.append(parsed[1:])
#
# print(row)
# return np.array(traj)
traj = []
count = 0
with open(self.base_dir + self.sequence + path) as data:
for line in data.readlines():
row = line.strip().split(' ')
if count == 0:
count += 1
continue
parsed = [
float(row[0]),
float(row[1]),
float(row[2]),
float(row[3]),
float(row[4]),
float(row[5]),
float(row[6]),
float(row[7])
]
traj.append(parsed[1:])
return np.array(traj)
def process(self, data):
tmp = []
#punctuation = string.punctuation
for i, d in enumerate(data.readlines()):
t = Tree.fromstring(d)
sentence = t.leaves()
#sentence = [w.lower() for w in sentence]
#sentence = [w for w in sentence if w not in punctuation]
self.count += Counter(sentence)
tmp.append(sentence)
return tmp
def read_net_dataidx_map(filename='./data/distribution_seed0/net_dataidx_map.txt'):
net_dataidx_map = {}
with open(filename, 'r') as data:
for x in data.readlines():
if '{' != x[0] and '}' != x[0] and ']' != x[0]:
tmp = x.split(':')
if '[' == tmp[-1].strip():
key = int(tmp[0])
net_dataidx_map[key] = []
else:
tmp_array = x.split(',')
net_dataidx_map[key] = [int(i.strip()) for i in tmp_array]
return net_dataidx_map
def read_data_distribution(filename='./data/distribution_seed0/distribution.txt'):
distribution = {}
with open(filename, 'r') as data:
for x in data.readlines():
if '{' != x[0] and '}' != x[0]:
tmp = x.split(':')
if '{' == tmp[1].strip():
first_level_key = int(tmp[0])
distribution[first_level_key] = {}
else:
second_level_key = int(tmp[0])
distribution[first_level_key][second_level_key] = int(tmp[1].strip().replace(',', ''))
return distribution
def readImg(listPath, index):
data = open(listPath, 'r')
imgPair = []
datas = data.readlines()
if len(index) == 2:
datas = datas[index[0]:index[1]]
print('loading ' + str(len(datas)) + ' imgs from ' + listPath)
for d in datas:
d = d.strip("\n")
d = d.strip("\000")
d = d.strip("\r")
d = d.split(" ")
imgPair.append([d[0], d[1]])
data.close()
return imgPair
def readIMU_File(self, path):
imu = []
count = 0
with open(self.base_dir + self.sequence + path) as data:
for line in data.readlines():
row = line.strip().split(' ')
if count == 0:
count += 1
continue
parsed = [
float(row[2]),
float(row[3]),
float(row[4]),
float(row[5]),
float(row[6]),
float(row[7])
]
imu.append(parsed)
return np.array(imu)
def readTrajectoryFile(self, path):
traj = []
count = 0
with open(self.base_dir + self.sequence + path) as data:
for line in data.readlines():
row = line.strip().split(' ')
# if count == 0:
# count += 1
# continue
parsed = [
float(row[2]),
float(row[3]),
float(row[4]),
float(row[5]),
float(row[6]),
float(row[7]),
float(row[8])
]
traj.append(parsed)
return np.array(traj)
opt = parser.parse_args()
if not opt.experiment_name:
opt.experiment_name = f'{opt.Transformation}-{opt.FeatureExtraction}-{opt.SequenceModeling}-{opt.Prediction}'
opt.experiment_name += f'-Seed{opt.manualSeed}'
# print(opt.experiment_name)
os.makedirs(f'./saved_models/{opt.experiment_name}', exist_ok=True)
""" vocab / character number configuration """
# if opt.sensitive:
# opt.character += 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
# opt.character += 'アイウエオカキクケコサシスセソタチツテトナニヌネノハヒフヘホマミムメモヤユヨワヲンバビブベボダヅデドザジズゼゾッャュョァィガギグゲゴ0123456789xmL本'
with open('char_list.txt', 'r', encoding='utf-8') as data:
datalist = data.readlines()
opt.character = datalist[0]
# opt.character = string.printable[:-6] # same with ASTER setting (use 94 char).
""" Seed and GPU setting """
# print("Random Seed: ", opt.manualSeed)
random.seed(opt.manualSeed)
np.random.seed(opt.manualSeed)
torch.manual_seed(opt.manualSeed)
torch.cuda.manual_seed(opt.manualSeed)
cudnn.benchmark = True
cudnn.deterministic = True
opt.num_gpu = torch.cuda.device_count()
# print('device count', opt.num_gpu)
def createDataset(checkValid=True):
"""
Create LMDB dataset for training and evaluation.
ARGS:
inputPath : input folder path where starts imagePath
outputPath : LMDB output path
gtFile : list of image path and label
checkValid : if true, check the validity of every image
"""
outputPath = './input/lmdb/Test'
os.makedirs(outputPath, exist_ok=True)
env = lmdb.open(outputPath, map_size=1099511627)
cache = {}
cnt = 1
gtFile = './input/gt_Test.txt'
with open(gtFile, 'r', encoding='UTF8') as data:
datalist = data.readlines()
nSamples = len(datalist)
for i in range(nSamples):
imagePath, label = datalist[i].strip('\n').split('\t')
# print('imagepath',imagePath)
inputPath = './input'
imagePath = os.path.join(inputPath, imagePath)
# print('imagepath2',imagePath)
# # only use alphanumeric data
# if re.search('[^a-zA-Z0-9]', label):
# continue
if not os.path.exists(imagePath):
# print('%s does not exist' % imagePath)
print("Test images not loaded")
continue
with open(imagePath, 'rb') as f:
imageBin = f.read()
if checkValid:
try:
if not checkImageIsValid(imageBin):
print('%s is not a valid image' % imagePath)
continue
except:
print('error occured', i)
with open(outputPath + '/error_image_log.txt', 'a') as log:
log.write('%s-th image data occured error\n' % str(i))
continue
imageKey = 'image-%09d'.encode() % cnt
labelKey = 'label-%09d'.encode() % cnt
cache[imageKey] = imageBin
cache[labelKey] = label.encode()
if cnt % 1000 == 0:
writeCache(env, cache)
cache = {}
# print('Written %d / %d' % (cnt, nSamples))
cnt += 1
nSamples = cnt - 1
cache['num-samples'.encode()] = str(nSamples).encode()
writeCache(env, cache)
#print('Created test dataset with %d samples' % nSamples)
print("%d samples loaded" % nSamples)
