def readIMU_File(self, path): # imu = [] # count = 0 # with open(self.base_dir + self.sequence + path) as csvfile: # spamreader = csv.reader(csvfile, delimiter=',', quotechar='|') # for row in spamreader: # if count == 0: # count += 1 # continue # parsed = [float(row[1]), float(row[2]), float(row[3]), # float(row[4]), float(row[5]), float(row[6])] # imu.append(parsed) # # return np.array(imu) imu = [] count = 0 with open(self.base_dir + self.sequence + path) as data: for line in data.readlines(): row = line.strip().split(' ') if count == 0: count += 1 continue parsed = [ float(row[1]), float(row[2]), float(row[3]), float(row[4]), float(row[5]), float(row[6]) ] imu.append(parsed) return np.array(imu)
def readTrajectoryFile(self, path): # traj = [] # with open(self.base_dir + self.sequence + path) as csvfile: # spamreader = csv.reader(csvfile, delimiter=',', quotechar='|') # for row in spamreader: # parsed = [float(row[0]), float(row[1]), float(row[2]), float(row[3]), # float(row[4]), float(row[5]), float(row[6]), float(row[7])] # traj.append(parsed[1:]) # # print(row) # return np.array(traj) traj = [] count = 0 with open(self.base_dir + self.sequence + path) as data: for line in data.readlines(): row = line.strip().split(' ') if count == 0: count += 1 continue parsed = [ float(row[0]), float(row[1]), float(row[2]), float(row[3]), float(row[4]), float(row[5]), float(row[6]), float(row[7]) ] traj.append(parsed[1:]) return np.array(traj)
def process(self, data): tmp = [] #punctuation = string.punctuation for i, d in enumerate(data.readlines()): t = Tree.fromstring(d) sentence = t.leaves() #sentence = [w.lower() for w in sentence] #sentence = [w for w in sentence if w not in punctuation] self.count += Counter(sentence) tmp.append(sentence) return tmp
def read_net_dataidx_map(filename='./data/distribution_seed0/net_dataidx_map.txt'): net_dataidx_map = {} with open(filename, 'r') as data: for x in data.readlines(): if '{' != x[0] and '}' != x[0] and ']' != x[0]: tmp = x.split(':') if '[' == tmp[-1].strip(): key = int(tmp[0]) net_dataidx_map[key] = [] else: tmp_array = x.split(',') net_dataidx_map[key] = [int(i.strip()) for i in tmp_array] return net_dataidx_map
def read_data_distribution(filename='./data/distribution_seed0/distribution.txt'): distribution = {} with open(filename, 'r') as data: for x in data.readlines(): if '{' != x[0] and '}' != x[0]: tmp = x.split(':') if '{' == tmp[1].strip(): first_level_key = int(tmp[0]) distribution[first_level_key] = {} else: second_level_key = int(tmp[0]) distribution[first_level_key][second_level_key] = int(tmp[1].strip().replace(',', '')) return distribution
def readImg(listPath, index): data = open(listPath, 'r') imgPair = [] datas = data.readlines() if len(index) == 2: datas = datas[index[0]:index[1]] print('loading ' + str(len(datas)) + ' imgs from ' + listPath) for d in datas: d = d.strip("\n") d = d.strip("\000") d = d.strip("\r") d = d.split(" ") imgPair.append([d[0], d[1]]) data.close() return imgPair
def readIMU_File(self, path): imu = [] count = 0 with open(self.base_dir + self.sequence + path) as data: for line in data.readlines(): row = line.strip().split(' ') if count == 0: count += 1 continue parsed = [ float(row[2]), float(row[3]), float(row[4]), float(row[5]), float(row[6]), float(row[7]) ] imu.append(parsed) return np.array(imu)
def readTrajectoryFile(self, path): traj = [] count = 0 with open(self.base_dir + self.sequence + path) as data: for line in data.readlines(): row = line.strip().split(' ') # if count == 0: # count += 1 # continue parsed = [ float(row[2]), float(row[3]), float(row[4]), float(row[5]), float(row[6]), float(row[7]), float(row[8]) ] traj.append(parsed) return np.array(traj)
opt = parser.parse_args() if not opt.experiment_name: opt.experiment_name = f'{opt.Transformation}-{opt.FeatureExtraction}-{opt.SequenceModeling}-{opt.Prediction}' opt.experiment_name += f'-Seed{opt.manualSeed}' # print(opt.experiment_name) os.makedirs(f'./saved_models/{opt.experiment_name}', exist_ok=True) """ vocab / character number configuration """ # if opt.sensitive: # opt.character += 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' # opt.character += 'アイウエオカキクケコサシスセソタチツテトナニヌネノハヒフヘホマミムメモヤユヨワヲンバビブベボダヅデドザジズゼゾッャュョァィガギグゲゴ0123456789xmL本' with open('char_list.txt', 'r', encoding='utf-8') as data: datalist = data.readlines() opt.character = datalist[0] # opt.character = string.printable[:-6] # same with ASTER setting (use 94 char). """ Seed and GPU setting """ # print("Random Seed: ", opt.manualSeed) random.seed(opt.manualSeed) np.random.seed(opt.manualSeed) torch.manual_seed(opt.manualSeed) torch.cuda.manual_seed(opt.manualSeed) cudnn.benchmark = True cudnn.deterministic = True opt.num_gpu = torch.cuda.device_count() # print('device count', opt.num_gpu)
def createDataset(checkValid=True): """ Create LMDB dataset for training and evaluation. ARGS: inputPath : input folder path where starts imagePath outputPath : LMDB output path gtFile : list of image path and label checkValid : if true, check the validity of every image """ outputPath = './input/lmdb/Test' os.makedirs(outputPath, exist_ok=True) env = lmdb.open(outputPath, map_size=1099511627) cache = {} cnt = 1 gtFile = './input/gt_Test.txt' with open(gtFile, 'r', encoding='UTF8') as data: datalist = data.readlines() nSamples = len(datalist) for i in range(nSamples): imagePath, label = datalist[i].strip('\n').split('\t') # print('imagepath',imagePath) inputPath = './input' imagePath = os.path.join(inputPath, imagePath) # print('imagepath2',imagePath) # # only use alphanumeric data # if re.search('[^a-zA-Z0-9]', label): # continue if not os.path.exists(imagePath): # print('%s does not exist' % imagePath) print("Test images not loaded") continue with open(imagePath, 'rb') as f: imageBin = f.read() if checkValid: try: if not checkImageIsValid(imageBin): print('%s is not a valid image' % imagePath) continue except: print('error occured', i) with open(outputPath + '/error_image_log.txt', 'a') as log: log.write('%s-th image data occured error\n' % str(i)) continue imageKey = 'image-%09d'.encode() % cnt labelKey = 'label-%09d'.encode() % cnt cache[imageKey] = imageBin cache[labelKey] = label.encode() if cnt % 1000 == 0: writeCache(env, cache) cache = {} # print('Written %d / %d' % (cnt, nSamples)) cnt += 1 nSamples = cnt - 1 cache['num-samples'.encode()] = str(nSamples).encode() writeCache(env, cache) #print('Created test dataset with %d samples' % nSamples) print("%d samples loaded" % nSamples)