def computeCorr(config, i, model, bcid, json, version):
args = [json, str(bcid), model, version]
names = ['cC'] + config.split('_') + [str(i), model, version]
submit('res/jobs/computeCorr.sh',
args=args,
names=names,
ma=ma,
mo=mo,
rt=3)
def integrateResiduals(config, i, model, bcid, json, version):
args = [json, str(bcid), model, version]
names = ['iR'] + config.split('_') + [str(i), model, version]
submit('res/jobs/integrateResiduals.sh',
args=args,
names=names,
ma=ma,
mo=mo,
vmem='5G',
rt=12)
def shapeFitter(config, i, model, bcid, json, version):
args = [json, str(bcid), model]
names = ['sF'] + config.split('_') + [str(i), model, version]
submit('res/jobs/shapeFitter_{0}.sh'.format(version),
args=args,
names=names,
ma=ma,
mo=mo,
rt=5,
test=test)
def submit_pipe():
data_path = Path('/run/media/andrey/Fast/FairFace/test/data')
csv_path = Path('/run/media/andrey/Fast/FairFace/test/evaluation_pairs/predictions.csv')
img_paths = list(data_path.iterdir())
cache_dir = Path('/run/media/andrey/Data/pipe_cache_test')
# img_matcher = config_resnet_matcher(img_paths, cache_dir, conf_thresh=0.97)
img_matcher = superglue_matcher(Path('/run/media/andrey/Data/sp_cache'))
detector = pipeline_detector(img_paths, cache_dir / 'detector', small_face=16)
experiment_names = ['ultimate7']
epochs = [10]
experiment_path = Path('experiments')
feature_extractors = [mxnet_feature_extractor(
cache_dir / f'extractor_{cur_exp}_{cur_epoch:04d}',
str(experiment_path / cur_exp / 'snapshots' / cur_exp),
cur_epoch, use_flip=True, ctx=mx.gpu(0))
for cur_exp, cur_epoch in zip(experiment_names, epochs)]
comparator = PipeMatcher(img_paths, cache_dir, img_matcher, detector, feature_extractors)
submit(data_path, csv_path, comparator, f'pipe_sp_ultimate7_097')
def compare(data_path: Path, csv_path: Path, experiment: str, epoch: int) -> None:
model_path = Path('experiments') / experiment / 'snapshots'
comparator = CompareModel(str(model_path / experiment), epoch, ctx=mx.gpu(0), use_flip=True)
comparator.metric = cosine
submit(data_path, csv_path, comparator, f'final_{experiment}_{epoch}_flip')
for i, bcid in enumerate(bcids):
for model, version in modelversion:
thejob(config, i, model, bcid, json, version)
else:
for config, i, model, version in confignummodel:
json = '{0}/res/hist/Fill{1}.json'.format(submit.cwd(), config)
with open(json) as f:
bcid = load(f)['bcids'][i]
thejob(config, i, model, bcid, json, version)
elif arg.startswith('makeHtml'):
fill = arg[8:]
names = ['html', fill]
submit('res/jobs/makeHtml_{0}.sh'.format(fill),
names=names,
ma=ma,
mo=mo,
rt=1)
elif arg in ('shapeFitter_v3', 'shapeFitter_v4'):
for config in configs:
json = '{0}/res/hist/Fill{1}.json'.format(submit.cwd(), config)
with open(json) as f:
bcids = load(f)['bcids']
for i, bcid in enumerate(bcids):
mymo = mo
for model, version in modelversion:
if version != arg[-2:]:
continue
for j in range(10, 10 + number):
if j == 5:
bili = getBilinear(order=0)
# near = getBilinear(order=0)
# for layer in range(3):
layer = 1
print('Generate layer %s'%layer)
eval_dataset = AGSDLoader(mode='test', layer=layer)
eval_dataloader = DataLoader(dataset=eval_dataset, batch_size=1)
model = AGSDNetV2().cuda()
state_dict = model.state_dict()
for n, p in torch.load('./checkpoint/agsdDem/epoch_40.pth', map_location=lambda storage, loc: storage).items():
if n in state_dict.keys():
state_dict[n].copy_(p)
result = []
model.eval()
with torch.no_grad():
for data in eval_dataloader:
inputs, _, _, sb, sn, dem, dem2 = data
inputs, sb, sn, dem, dem2 = inputs.cuda().float(), sb.cuda().float(), sn.cuda().float(), dem.cuda().float(), dem2.cuda().float()
with torch.no_grad():
preds, _ = model(inputs, dem, sb, sn, dem2)
result.append(preds.data.cpu().numpy())
result = np.concatenate(result, axis=0).transpose((2,3,1,0))
bili[:,:,layer:layer+1,:] = result
# bili[:,:,2:3,:] = near[:,:,2:3,:]
submit(bili, 'agsdDem.mat')
from lib.bilinear import getBilinear from lib.submit import submit # submit(getBilinear(order=3), 'bilinear.mat') submit(getBilinear(order=0), 'nearest.mat')
'/run/media/andrey/Fast/FairFace/data/val/evaluation_pairs/predictions.csv'
)
data_path = Path('/run/media/andrey/Fast/FairFace/fixed_data/val/data')
all_imgs = list(data_path.iterdir())
algorithm = cluster.AgglomerativeClustering(
n_clusters=None,
affinity='cosine',
memory='/run/media/andrey/Data/cluster_cache/',
linkage='complete',
distance_threshold=0.92)
norm_median = False
median_alpha = 0.5
cur_exp = 'ultimate5'
model_path = Path('experiments') / cur_exp / 'snapshots'
cur_epoch = len(list(model_path.iterdir())) - 1
comparator = CompareModel(str(model_path / cur_exp),
cur_epoch,
use_flip=True,
ctx=mx.gpu(0))
# metric = metrics.euclidean
# cluster_comparator_eu = config_median_comparator(comparator, partial(cluster_sklearn, algorithm=algorithm),
# val_data['img_path'], metric, norm_median, median_alpha)
metric = metrics.cosine
cluster_comparator_co = config_median_comparator(
comparator, partial(cluster_sklearn, algorithm=algorithm), all_imgs,
metric, norm_median, median_alpha)
submit_name = 'ultimate5+cluster_cosine_f'
submit(data_path, val_csv, comparator, submit_name)
import numpy as np
import mxnet as mx
from pathlib import Path
from lib.compare import CompareModel
from lib.submit import submit
if __name__ == '__main__':
comparator = CompareModel('/home/andrey/.insightface/models/VGG2-ResNet50-Arcface/model', 0,
ctx=mx.gpu(0), use_flip=True)
val_dir = Path('/run/media/andrey/Fast/FairFace/test_wide/data')
val_csv = Path('/run/media/andrey/Fast/FairFace/test/evaluation_pairs/predictions.csv')
submit_name = 'final_arcface_vgg_flip_allval'
submit(val_dir, val_csv, comparator, submit_name)
cudnn.benchmark = True
device = torch.device('cuda: 7' if torch.cuda.is_available() else 'cpu')
eval_dataset = SRCNNLoader(mode='test')
eval_dataloader = DataLoader(dataset=eval_dataset, batch_size=1)
bili = getBilinear(order=0)
key_value = {1: 5} #, 2:5}
for layer, epoch in key_value.items():
print('Generate layer %s' % layer)
model = SRCNN().to(device)
state_dict = model.state_dict()
for n, p in torch.load(
'./checkpoint/srcnn%s/epoch_%s.pth' % (layer, epoch),
map_location=lambda storage, loc: storage).items():
if n in state_dict.keys():
state_dict[n].copy_(p)
result = []
model.eval()
with torch.no_grad():
for data in eval_dataloader:
inputs, _, _ = data
inputs = inputs.to(device)
with torch.no_grad():
preds = model(inputs)
result.append(preds.data.cpu().numpy())
result = np.concatenate(result, axis=0).transpose((2, 3, 1, 0))
bili[:, :, layer:layer + 1, :] = result
submit(bili, 'srcnn_1_near.mat')
from lib.submit import submit
if __name__ == '__main__':
cudnn.benchmark = True
device = torch.device('cuda: 7' if torch.cuda.is_available() else 'cpu')
eval_dataset = SRCNNLoader3D(mode='test')
eval_dataloader = DataLoader(dataset=eval_dataset, batch_size=1)
bili = getBilinear()
model = SRCNN3D().to(device)
state_dict = model.state_dict()
for n, p in torch.load('./checkpoint/srcnn3d/epoch_%s.pth' % (4),
map_location=lambda storage, loc: storage).items():
if n in state_dict.keys():
state_dict[n].copy_(p)
result = []
model.eval()
with torch.no_grad():
for data in eval_dataloader:
inputs, _, _ = data
inputs = inputs.to(device)
with torch.no_grad():
preds = model(inputs)
result.append(preds.data.cpu().numpy())
result = np.concatenate(result, axis=0).squeeze(1).transpose((2, 3, 1, 0))
bili[:, :, :3, :] = result
submit(bili, 'srcnn3d.mat')