def __getitem__(self,idx): idx%=self.size inputs=cv2.imread(PJ(self.imgdir,self.namelist[idx]),0) targets=cv2.imread(PJ(self.labdir,self.namelist[idx]),0) crop=self.transforms(np.concatenate((inputs[...,np.newaxis],targets[...,np.newaxis]),axis=-1)) inputs=crop[0:1] targets=(crop[1:2]<=self.th).float() return inputs,(inputs,targets)
def __getitem__(self,idx): idx=idx%self.size inputs=cv2.imread(PJ(self.imgdir,self.namelist[idx]),0) targets=cv2.imread(PJ(self.labdir,self.namelist[idx]),0) if self.train: crop=self.transforms(np.concatenate((inputs[...,np.newaxis],targets[...,np.newaxis]),axis=-1)) inputs=crop[0:1].unsqueeze(0) targets=(crop[1:2]<=self.th).float().unsqueeze(0) else: h,w=inputs.shape H,W=h//self.cell,w//self.cell inputs=self.transforms(inputs) targets=(self.transforms(targets)<=self.th).float() inputs=inputs.reshape(H,self.cell,W,self.cell).permute(0,2,1,3).reshape(H*W,self.cell,self.cell).unsqueeze(1) targets=targets.reshape(H,self.cell,W,self.cell).permute(0,2,1,3).reshape(H*W,self.cell,self.cell).unsqueeze(1) return inputs,targets
def __init__(self, data_root, data_file, class2idx, transform=None): self.data_root = PJ(os.getcwd(), *data_root) self.class2idx = class2idx self.transform = transform # Load data path images = [] dir = os.path.expanduser(PJ(self.data_root, data_file)) for target in sorted(self.class2idx.keys()): d = os.path.join(dir, target) if not os.path.isdir(d): continue for root, _, fnames in sorted(os.walk(d)): for fname in sorted(fnames): path = os.path.join(root, fname) item = [path, target] images.append(item) self.data = images
transforms = T.Compose([ T.ToPILImage(), T.ToTensor(), ]) dataset = Dataset(cfg, mode, transforms) loader = torch.utils.data.DataLoader(dataset, batch_size=cfg['batch_size'], shuffle=cfg['shuffle'], num_workers=cfg['num_workers']) return loader if __name__ == "__main__": mode = 'test' direction = { 'test': PJ("D:\\Data", "ISBI-2012-EM", "new_test_set"), "train": PJ("D:\\Data", "ISBI-2012-EM", "new train set") } cfg = { "file_name": "ISBI", "num_workers": 1, "th": 0.6, 'repeat_time': 10, "batch_size": 1, "shuffle": True, 'direction': direction[mode] } d = iter(dataloader(cfg, mode)) for kkk in range(10): i, t = next(d) print(t.sum() / 512 / 512)
import os from os.path import join as PJ import scipy.io as sio from pandas import DataFrame as df import yaml # Setting DATASET = "apy" ROOT = PJ("..", "dataset") XLSA17 = PJ(ROOT, "xlsa17", "data", DATASET) CONCEPT = PJ(ROOT, DATASET, "list", "concepts", "concepts.txt") CONCEPT_LIST = PJ(ROOT, DATASET, "list", "concepts") SPLIT_LIST = PJ(ROOT, DATASET, "list", "train_test") ATT_SPLITS = sio.loadmat(PJ(XLSA17, "att_splits.mat")) RES101 = sio.loadmat(PJ(XLSA17, "res101.mat")) # check path exist if not os.path.isdir(PJ(ROOT, DATASET, "list")): os.makedirs(PJ(ROOT, DATASET, "list")) if not os.path.isdir(CONCEPT_LIST): os.makedirs(CONCEPT_LIST) if not os.path.isdir(SPLIT_LIST): os.makedirs(SPLIT_LIST) # reorganize data img_files = [filter(None, i[0][0].split('/')) for i in RES101['image_files']] img_files = [PJ(*list(i)[5:]) for i in img_files]
######################################## # Environment and Experiment setting ######################################## # Parse arguments parser = argparse.ArgumentParser() parser.add_argument('--exp', '-e', type=str, default='exp1_ResNet18') parser.add_argument('--model_weights', '-w', type=str, default='') args = parser.parse_args() # Load experiment config config_path = f"./configs/{args.exp}.yaml" config = config(config_path) exp_name = config['exp_name'] print(f"EXP: {exp_name}") save_root = PJ(f"./results", exp_name) # Show some experiment info model_weights = args.model_weights print(f"Model: {config['model']}, Weights: {model_weights}") print(f"Pretrained: {config['pretrained']}, Freeze: {config['freeze']}") ######################################## # Data loader ######################################## class2idx = config['class2idx'] idx2class = {class2idx[k]: k for k in class2idx.keys()} # Dataset transform = data_transform(config, train=False) testset = CIFAR10Dataset(config['data_root'], config['test_file'], class2idx, transform)
import re import pandas as pd import numpy as np from os.path import join as PJ import scipy.io as sio from gensim.models import KeyedVectors DATASET = "cub" concept_vec_filename = "concepts_new.txt" # concept_vec_filename = "concepts_old.txt" weight_path = "self-enwiki-gensim-normed-win10.bin" # weight_path = "enwiki-gensim-normed.bin" ROOT = PJ("..", "dataset") concept_filename = PJ(ROOT, DATASET, "list", "concepts", "concepts.txt") concept_vec_filename = PJ(ROOT, DATASET, "list", "concepts", concept_vec_filename) weight_path = PJ(ROOT, "enwiki", weight_path) ATT_SPLITS = sio.loadmat(PJ(ROOT, "xlsa17", "data", DATASET, "att_splits.mat")) RES101 = sio.loadmat(PJ(ROOT, "xlsa17", "data", DATASET, "res101.mat")) setting = { 'apy': { 'replace_word': { 'diningtable': 'tables', 'pottedplant': 'houseplant', # 'tvmonitor': 'tv' 'tvmonitor': 'flat_panel_display' }
for tn in STATE['split_list']: df = datasets[tn].data.iloc[:, 1:].sum(axis=0) print(tn + ": " + str(df[df > 0].skew())) train_loader = DataLoader(datasets['trainval'], batch_size=CONFIG['train_batch_size'], shuffle=True) test_loaders = {tn: DataLoader(datasets[tn], batch_size=CONFIG['test_batch_size'], shuffle=False) for tn in STATE['split_list'][1:]} ########################################################################################## # experiment for n times for exp_times in range(CONFIG['exp_times']): SAVE_PATH = PJ('.', 'runs_test', DATASET, EXP_NAME, str(exp_times)) writer = SummaryWriter(PJ(SAVE_PATH)) # set experiment type: classifier / transformer if CONFIG['type'] == "classifier": model = classifier(backbone=CONFIG['model'], k=CONFIG['k'], d=CONFIG['d'][CONFIG['concepts']][DATASET], pretrained=CONFIG['pretrained'], freeze=CONFIG['freeze']) elif CONFIG['type'] == "transformer": model = transformer(backbone=CONFIG['model'], linear=CONFIG['linear'], k=CONFIG['k'], d=CONFIG['d'][CONFIG['concepts']][DATASET], pretrained=CONFIG['pretrained'], freeze=CONFIG['freeze']) else: assert False, "Must Assign the model type: classifier or transformer"
import pymarc #Please load the supplied version of pymarc (install using pip install ./) import numpy as np import ujson import json import bgzf import struct import os from tqdm.auto import tqdm from os.path import join as PJ #Set to the location where INSPIRE dump is. inspireDumpPath = PJ("..", "Data", "INSPIRE", "20210108") processedDataPath = PJ(inspireDumpPath, "Processed") os.makedirs(processedDataPath, exist_ok=True) def getTagSubtag(data, subtag, onErrorParent=True): if (isinstance(data, dict)): value = None # try: if (subtag not in data): value = None else: value = data[subtag] # except:
import os from os.path import join as PJ import scipy.io as sio from pandas import DataFrame as df # Setting DATASET = "awa2" ROOT = PJ("..", "dataset") XLSA17 = PJ(ROOT, "xlsa17", "data", DATASET) CONCEPT = PJ(ROOT, DATASET, "list", "concepts", "concepts.txt") CONCEPT_LIST = PJ(ROOT, DATASET, "list", "concepts") SPLIT_LIST = PJ(ROOT, DATASET, "list", "train_val") ATT_SPLITS = sio.loadmat(PJ(XLSA17, "att_splits.mat")) RES101 = sio.loadmat(PJ(XLSA17, "res101.mat")) # check path exist if not os.path.isdir(PJ(ROOT, DATASET, "list")): os.makedirs(PJ(ROOT, DATASET, "list")) if not os.path.isdir(CONCEPT_LIST): os.makedirs(CONCEPT_LIST) if not os.path.isdir(SPLIT_LIST): os.makedirs(SPLIT_LIST) # reorganize data img_files = [filter(None, i[0][0].split('/')) for i in RES101['image_files']] img_files = [PJ(*list(i)[5:]) for i in img_files]
import os from os.path import join as PJ import scipy.io as sio from pandas import DataFrame as df import yaml from sklearn.preprocessing import LabelBinarizer # Setting DATASET = "awa2" ROOT = PJ("..", "dataset") XLSA17 = PJ(ROOT, "xlsa17", "data", DATASET) CONCEPT = PJ(ROOT, DATASET, "list", "concepts", "concepts.txt") CONCEPT_LIST = PJ(ROOT, DATASET, "list", "concepts") SPLIT_LIST = { "train_test": PJ(ROOT, DATASET, "list", "train_test"), "train_val": PJ(ROOT, DATASET, "list", "train_test") } ATT_SPLITS = sio.loadmat(PJ(XLSA17, "att_splits.mat")) RES101 = sio.loadmat(PJ(XLSA17, "res101.mat")) #################### # check path exist # #################### if not os.path.isdir(PJ(ROOT, DATASET, "list")): os.makedirs(PJ(ROOT, DATASET, "list")) if not os.path.isdir(CONCEPT_LIST): os.makedirs(CONCEPT_LIST)
if __name__ == '__main__': from models.network import * import argparse import os from os.path import join as PJ from RandAugment import RandAugment from custom_data_io import myset import torch.optim as optim from DML import DML parser = argparse.ArgumentParser() parser.add_argument('-e', '--exp', type=str, default='exp_1') args = parser.parse_args() config_path = PJ(os.getcwd(), f"{args.exp}.yaml") config = config(config_path) exp_name = config['exp_name'] print(f"EXP: {exp_name}") train_transforms = transforms.Compose([transforms.Resize((385, 505)), transforms.RandomHorizontalFlip(), transforms.RandomVerticalFlip(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])]) test_transforms = transforms.Compose([transforms.Resize((385, 505)), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])]) train_transforms.transforms.insert(0, RandAugment(2, 14))