def extract_dataset(option, file_path): data_dir = '' info_file = '' if option == 'option1': try: find_image_files_folder_per_class(data_dir) except AssertionError: return False elif option == 'option2': try: find_image_files_from_file(data_dir, info_file) except AssertionError: return False elif option == 'option3': pass return True
def test_upload(self, request): test_file = request.files['input_file'] filename = test_file.filename dataset_test_path = os.path.join( self._dataset.get_dataset_path().replace('train', 'test'), filename.split('.')[0]) path_file = os.path.join(dataset_test_path, filename) os.makedirs(dataset_test_path, exist_ok=True) test_file.save(path_file) try: if path_file.endswith('.npz'): try: _, test_data = find_dataset_from_numpy(path_file, requires_y=False, only_test=True) if test_data is None: tree_remove(dataset_test_path) return 'The file contents are not valid.' np.savez(path_file, x=test_data[0], y=test_data[1]) open(os.path.join(dataset_test_path, '.option0'), 'w') # NUMPY FILE return 'ok' except KeyError: tree_remove(dataset_test_path) return "The file contents are not valid." else: if not unzip(path_file, dataset_test_path): return "The file contents already exists." os.remove(path_file) if find_images_test_file(dataset_test_path): open(os.path.join(dataset_test_path, '.option1'), 'w') # ONLY IMAGES else: try: f, n, c = find_image_files_folder_per_class( dataset_test_path, require_all=False) assert len(c) == len(self.get_target_labels()) open(os.path.join(dataset_test_path, '.option2'), 'w') # FOLDER PER CLASS except AssertionError: try: info_file = [ f for f in os.listdir(dataset_test_path) if f.startswith('test.') or f.startswith('labels.') ] assert len(info_file) == 1 info_path = os.path.join(dataset_test_path, info_file[0]) f, n, c = find_image_files_from_file( dataset_test_path, info_path, require_all=False) assert len(c) == len(self.get_target_labels()) open(os.path.join(dataset_test_path, '.option3'), 'w') # LABELS.TXT os.rename( info_path, os.path.join(dataset_test_path, 'labels.txt')) except AssertionError: tree_remove(dataset_test_path) return "The file contents are not valid." except ValueError: tree_remove(dataset_test_path) return "The file contents are not valid." return 'ok'
def test_request(self, request): has_targets = True df_test = {} test_filename = [] labels = [] if 'filename' in request.get_json(): if get_filename(request) == 'TEST FROM SPLIT': test_filename = self._dataset.get_test_split_images() df_test = { self._dataset.get_targets()[0]: self._dataset.get_test_split_labels() } else: test_path = os.path.join( self._dataset.get_dataset_path().replace('train', 'test'), get_filename(request)) option = [ f for f in os.listdir(test_path) if f.startswith('.option') ][0] if option == '.option0': test_path = os.path.join(test_path, get_filename(request) + '.npz') data = np.load(test_path) test_filename = data['x'] if 'y' in data: labels = data['y'] assert len(data['y'].shape) < 3 # TODO else: return False, test_filename, None, None elif option == '.option1': test_filename = [ os.path.join(test_path, t) for t in os.listdir(test_path) if not t.startswith('.') ] return False, test_filename, None, None elif option == '.option2': all_classes = [ c for c in os.listdir(test_path) if not c.startswith('.') ] for cl in all_classes: list_files = [ os.path.join(test_path, cl, f) for f in os.listdir(os.path.join(test_path, cl)) if not f.startswith('.') ] test_filename += list_files labels += [cl] * len(list_files) elif option == '.option3': labels_file = [ os.path.join(test_path, t) for t in os.listdir(test_path) if t.startswith('labels.') ] # print(labels_file) test_filename, labels, _ = find_image_files_from_file( test_path, labels_file[0], require_all=False) df_test[self._dataset.get_targets()[0]] = labels return has_targets, test_filename, df_test, None return False
def new_image_dataset(USER_ROOT, username, option, file): if isinstance(file, str): return False dataset_name = file.filename.split('.')[0] dataset_name, dataset_path = check_dataset_path(USER_ROOT, username, dataset_name) open(os.path.join(dataset_path, option_map[option]), 'w') dataset_test_path = os.path.join(dataset_path, 'test') os.makedirs(dataset_test_path, exist_ok=True) train_path = os.path.join(dataset_path, 'train') os.makedirs(train_path, exist_ok=True) filename = secure_filename(file.filename) path_file = os.path.join(dataset_path, filename) file.save(path_file) if option == 'option3': try: train_data, test_data = find_dataset_from_numpy(path_file) np.savez(os.path.join(train_path, filename), x=train_data[0], y=train_data[1]) if test_data: os.makedirs(os.path.join(dataset_test_path, dataset_name), exist_ok=True) np.savez(os.path.join(dataset_test_path, dataset_name, dataset_name + '.npz'), x=test_data[0], y=test_data[1]) open(os.path.join(dataset_test_path, dataset_name, '.option0'), 'w') # NUMPY FILE return dataset_name except Exception as e: tree_remove(dataset_path) raise e if not check_zip_file(path_file): tree_remove(dataset_path) raise ValueError('Invalid file.') unzip(path_file, train_path) try: if option == 'option1': if 'train' in os.listdir(train_path): rename(os.path.join(train_path, 'train'), train_path) find_image_files_folder_per_class(train_path) if 'test' in os.listdir(train_path): dataset_test_path = os.path.join(dataset_test_path, dataset_name) os.makedirs(dataset_test_path, exist_ok=True) rename(os.path.join(train_path, 'test'), dataset_test_path) find_image_files_folder_per_class(dataset_test_path, require_all=False) open(os.path.join(dataset_test_path, '.option2'), 'w') elif option == 'option2': info_file = [f for f in os.listdir(train_path) if f.startswith('labels.') or f.startswith('train.')] assert len(info_file) == 1 os.rename(os.path.join(train_path, info_file[0]), os.path.join(train_path, 'labels.txt')) find_image_files_from_file(train_path, os.path.join(train_path, 'labels.txt')) info_test_file = [f for f in os.listdir(train_path) if f.startswith('test.')] if len(info_test_file) == 1: find_image_files_from_file(train_path, os.path.join(train_path, info_test_file[0]), require_all=False) dataset_test_path = os.path.join(dataset_test_path, dataset_name) os.makedirs(dataset_test_path, exist_ok=True) args = {} if not has_header(os.path.join(train_path, info_test_file[0])): args['header'] = None df = pd.read_csv(os.path.join(train_path, info_test_file[0]), sep=None, engine='python', **args) filenames = df[df.columns[0]].values if not os.path.isfile(filenames[0]): filenames = [os.path.join(train_path, f) for f in filenames] for f in filenames: os.rename(f, os.path.join(dataset_test_path, os.path.basename(f))) os.rename(os.path.join(train_path, info_test_file[0]), os.path.join(dataset_test_path, 'labels.txt')) open(os.path.join(dataset_test_path, '.option3'), 'w') # NUMPY FILE except AssertionError as e: tree_remove(dataset_path) raise e return dataset_name