def empirical_evaluation(G, interval):
"""Empirical evaluation
Plot the computation time (y-axis) given the number of nodes (x-axis)
of subgraphs, for each interval of nodes.
"""
X = list(range(interval, len(G.nodes) + 1, interval))
Y_our = []
Y_nx = []
for i in X:
sub_G = G.subgraph(list(G.nodes)[0:i])
with Timer(f"Our betweenness_centrality ({i} nodes)") as t:
bc.betweenness_centrality(sub_G)
Y_our.append(t.get_time())
with Timer(f"NetworkX betweenness_centrality ({i} nodes)") as t:
nx.betweenness_centrality(sub_G)
Y_nx.append(t.get_time())
print("---")
plt.figure()
plt.title("Computing time of the function Betweenness centrality")
plt.xlabel("Number of nodes")
plt.ylabel("Time [s]")
plt.plot(X, Y_our, label="Our implementation")
plt.plot(X, Y_nx, label="NetworkX implementation")
plt.legend()
def convert_genotypes(self):
chunk_size = self.split_size
if chunk_size is None:
raise ValueError(
'CONVERTER_SPLIT_SIZE does not define in config file!')
G = np.array([])
#self.reader.folder.processed=0
while True:
with Timer() as t:
G = self.reader.folder.get_bed(chunk_size)
if isinstance(G, type(None)):
break
print('Time to read {} SNPs is {} s'.format(G.shape[0], t.secs))
self.write_data('gen')
atom = tables.Int8Atom()
self.genotype = self.h5_gen_file.create_carray(
self.h5_gen_file.root,
'genotype',
atom, (G.shape),
title='Genotype',
filters=self.pytable_filters)
with Timer() as t:
self.genotype[:] = G
print('Time to write {} SNPs is {} s'.format(G.shape[0], t.secs))
self.h5_gen_file.close()
G = None
gc.collect()
def startup(self, now, persistant):
state_machine.State.startup(self, now, persistant)
self.number_plateform_save = LandingConfig.numberOfPlateforms
self.gravity_save = LanderConfig.GRAVITY
LanderConfig.GRAVITY = 0
LandingConfig.numberOfPlateforms = 0
self.draw_credits = False
self.timer_delay = Timer(CreditConfig.DELAY_BETWEEN_CREDITS)
self.land = Landing()
self.aircraft = Lander()
self.aircraft.fuel = 1000
self.timer_display_message = Timer(CreditConfig.CREDIT_DURATION)
self.aircraft_team = pygame.sprite.Group(self.aircraft)
self.aircraft.orientation = 0
self.credit_index = 0
def __init__(self, kS, kV, trackwidth, trajectory):
'''
Creates a controller for following a PathWeaver trajectory.
__init__(self, kS: Volts, kV: Volts * Seconds / Meters, trackwidth: Meters, trajectory: wpilib.trajectory.Trajectory)
:param kS: The kS gain determined by characterizing the Robot's drivetrain
:param kV: The kV gain determined by characterizing the Robot's drivetrain
:param trackwidth: The horizontal distance between the left and right wheels of the tank drive.
:param trajectory: The trajectory to follow. This can be generated by PathWeaver, or made by hand.
'''
self.kS = kS
self.kV = kV
self.trajectory = trajectory
self.odometry = DifferentialDriveOdometry(
Rotation2d(radians(0)), self.trajectory.initialPose())
self.ramsete = RamseteController(2, 0.7)
self.drive_kinematics = DifferentialDriveKinematics(trackwidth)
self.are_wheel_speeds_zero = False
self.timer = Timer()
def test_chip(test_set, rebuilder, transform, save_dir):
_t = Timer()
cost_time = list()
for type in test_set.test_dict:
img_list = test_set.test_dict[type]
if not os.path.exists(os.path.join(save_dir, type)):
os.mkdir(os.path.join(save_dir, type))
for k, path in enumerate(img_list):
image = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
_t.tic()
ori_img, input_tensor = transform(image)
out = rebuilder.inference(input_tensor)
re_img = out[0]
s_map = ssim_seg(ori_img, re_img, win_size=11, gaussian_weights=True)
mask = seg_mask(s_map, threshold=32)
inference_time = _t.toc()
cat_img = np.concatenate((ori_img, re_img, mask), axis=1)
cv2.imwrite(os.path.join(save_dir, type, '{:d}.png'.format(k)), cat_img)
cost_time.append(inference_time)
if (k+1) % 20 == 0:
print('{}th image, cost time: {:.1f}'.format(k+1, inference_time*1000))
_t.clear()
# calculate mean time
cost_time = np.array(cost_time)
cost_time = np.sort(cost_time)
num = cost_time.shape[0]
num90 = int(num*0.9)
cost_time = cost_time[0:num90]
mean_time = np.mean(cost_time)
print('Mean_time: {:.1f}ms'.format(mean_time*1000))
def benchmark_girvan_newman(G):
print("Starting benchmark_girvan_newman...")
max_iteration_level = 10
Y_our = []
Y_nx = []
our_it = our_girvan_newman(G)
nx_it = nx_girvan_newman(G)
our_it = itertools.islice(our_it, max_iteration_level)
nx_it = itertools.islice(nx_it, max_iteration_level)
i = 0
while True:
try:
with Timer(f"Starting iteration {i} on our girvan newman") as t:
next(our_it)
Y_our.append(t.get_time())
with Timer(f"Starting iteration {i} on nx girvan newman") as t:
next(nx_it)
Y_nx.append(t.get_time())
except StopIteration:
break
i += 1
X = list(range(len(Y_our)))
plt.figure()
plt.title("Time over iteration for executing girvan newman")
plt.xlabel("Iteration level []")
plt.ylabel("Time [s]")
plt.plot(X, Y_our, label="Our implementation")
plt.plot(X, Y_nx, label="Networkx implementation")
plt.legend()
plt.savefig("benchmark_girvan_newman.png")
Y_our_cumulative = get_cumulative_array(Y_our)
Y_nx_cumulative = get_cumulative_array(Y_nx)
plt.figure()
plt.title("Cumulative time over iteration for executing girvan newman")
plt.xlabel("Iteration level []")
plt.ylabel("Time [s]")
plt.plot(X, Y_our_cumulative, label="Our implementation")
plt.plot(X, Y_nx_cumulative, label="Networkx implementation")
plt.legend()
plt.savefig("benchmark_girvan_newman_cumulative.png")
def __init__(self, *args, **kwargs):
'''
Do not overload me!
'''
self.testStart(*args, **kwargs)
self.TEST_TIMER = Timer()
self.PERIODIC_DONE = False
self.END_DONE = False
def __init__(self):
'''
'''
self.feed_motor = SparkMax(MAGAZINE_FEED_MOTOR)
self.left_agitator = SparkMax(MAGAZINE_LEFT_MOTOR)
self.right_agitator = SparkMax(MAGAZINE_RIGHT_MOTOR)
# Timer used to get motor up to speed
self.timer = Timer()
def test_chip(test_set, rebuilder, transform, save_dir, configs):
_t = Timer()
cost_time = list()
iou_list={}
s_map_list=list()
for type in test_set.test_dict:
img_list = test_set.test_dict[type]
if not os.path.exists(os.path.join(save_dir, type)):
os.mkdir(os.path.join(save_dir, type))
os.mkdir(os.path.join(save_dir, type, 'ori'))
os.mkdir(os.path.join(save_dir, type, 'gen'))
os.mkdir(os.path.join(save_dir, type, 'mask'))
if not os.path.exists(os.path.join(save_dir, type,'ROC_curve')):
os.mkdir(os.path.join(save_dir, type, 'ROC_curve'))
for k, path in enumerate(img_list):
name= path.split('/')[-1]
image = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
_t.tic()
ori_img, input_tensor = transform(image)
out = rebuilder.inference(input_tensor)
re_img = out[0]
s_map = ssim_seg(ori_img, re_img, win_size=11, gaussian_weights=True)
_h, _w = image.shape
s_map_save = cv2.resize(s_map, (_w, _h))
s_map_list.append(s_map_save.reshape(-1,1))
mask = seg_mask(s_map, threshold=128)
inference_time = _t.toc()
if configs['db']['resize'] == [832, 832]:
#cat_img = np.concatenate((ori_img[32:-32,32:-32], re_img[32:-32,32:-32], mask[32:-32,32:-32]), axis=1)
cv2.imwrite(os.path.join(save_dir, type, 'ori', 'mask{:d}.png'.format(k)), ori_img[32:-32,32:-32])
cv2.imwrite(os.path.join(save_dir, type, 'gen', 'mask{:d}.png'.format(k)), re_img[32:-32,32:-32])
cv2.imwrite(os.path.join(save_dir, type, 'mask', 'mask{:d}.png'.format(k)), mask[32:-32,32:-32])
elif configs['db']['resize'] == [768, 768]:
cv2.imwrite(os.path.join(save_dir, type, 'ori', 'mask{:d}.png'.format(k)), ori_img)
cv2.imwrite(os.path.join(save_dir, type, 'gen', 'mask{:d}.png'.format(k)), re_img)
cv2.imwrite(os.path.join(save_dir, type, 'mask', 'mask{:d}.png'.format(k)), mask)
elif configs['db']['resize'] == [256, 256]:
cv2.imwrite(os.path.join(save_dir, type, 'ori', name), ori_img)
cv2.imwrite(os.path.join(save_dir, type, 'gen', name), re_img)
cv2.imwrite(os.path.join(save_dir, type, 'mask', name), mask)
else:
raise Exception("invaild image size")
#cv2.imwrite(os.path.join(save_dir, type, '{:d}.png'.format(k)), cat_img)
cost_time.append(inference_time)
if (k+1) % 20 == 0:
print('{}th image, cost time: {:.1f}'.format(k+1, inference_time*1000))
_t.clear()
torch.save(s_map_list,os.path.join(save_dir) + '/s_map.pth')
# calculate mean time
cost_time = np.array(cost_time)
cost_time = np.sort(cost_time)
num = cost_time.shape[0]
num90 = int(num*0.9)
cost_time = cost_time[0:num90]
mean_time = np.mean(cost_time)
print('Mean_time: {:.1f}ms'.format(mean_time*1000))
test_set.eval(save_dir)
def startup(self, now, persistant):
self.state_machine.setup_states({
'GAME': Game(),
'CREDITS': Credits()
}, 'CREDITS')
self.state_machine.state.startup(now, persistant)
self.should_flip = False
self.blink = True
self.timer_delay = Timer(1000)
def __init__(self):
self.clockwise_limit_switch = DigitalInput(
TURRET_CLOCKWISE_LIMIT_SWITCH)
self.counterclockwise_limit_switch = DigitalInput(
TURRET_COUNTERCLOCKWISE_LIMIT_SWITCH)
self.turn_motor = SparkMax(TURRET_TURN_MOTOR)
self.turn_pid = PIDController(0.4, 0.001, 0.02)
self.shoot_motor_1 = Falcon(TURRET_SHOOT_MOTORS[0])
self.shoot_motor_2 = Falcon(TURRET_SHOOT_MOTORS[1])
self.timer = Timer()
self.limelight = Limelight()
def benchmark_edge_betweenness_centrality(G):
print("Starting benchmark_edge_betweenness_centrality...")
X = list(range(0, len(G.nodes), 5))
Y_our = []
Y_nx = []
for i in X:
sub_G = G.subgraph(list(G.nodes)[0:i])
with Timer("Starting our edge_betweenness_centrality") as t:
our_edge_betweenness_centrality(sub_G)
Y_our.append(t.get_time())
with Timer("Starting nx edge_betweenness_centrality") as t:
nx_edge_betweenness_centrality(sub_G)
Y_nx.append(t.get_time())
plt.figure()
plt.title("Time over graph size for executing edge_betweenness_centrality")
plt.xlabel("Iteration level []")
plt.ylabel("Time [s]")
plt.plot(X, Y_our, label="Our implementation")
plt.plot(X, Y_nx, label="Networkx implementation")
plt.legend()
plt.savefig("benchmark_edge_betweenness_centrality.png")
def _script_for_patch_feature(head,
tail,
device,
dataset='mscoco',
data_split='train'):
data_dir = osp.join(DATA_ROOT, dataset)
# set the model path here
prototxt = osp.join(CAFFE_ROOT, 'models', 'ResNet',
'ResNet-152-deploy.prototxt')
caffemodel = osp.join(CAFFE_ROOT, 'models', 'ResNet',
'ResNet-152-model.caffemodel')
# load network
net, transformer = load_network(prototxt, caffemodel, device)
# prepare image files
cap_file = osp.join(data_dir, 'captions_{}.json'.format(data_split))
image_ids = json.load(open(cap_file, 'r'))['image_ids']
im_files = [
'COCO_{}2014_'.format(data_split) + str(i).zfill(12) + '.jpg'
for i in image_ids
]
im_files = [osp.join(data_dir, data_split + '2014', i) for i in im_files]
with h5py.File(osp.join(data_dir, 'bbox.h5'), 'r') as f:
bbox = np.array(f[data_split])
# initialize h5 file
save_file = osp.join(data_dir, 'features', 'features_30res.h5')
with h5py.File(save_file) as f:
if data_split not in f:
f.create_dataset(data_split, [len(im_files), 30, 2048], 'float32')
# computing
timer = Timer()
print '\n\n... computing'
for i in xrange(head, tail):
timer.tic()
im = caffe.io.load_image(im_files[i]) # (h, w, c)
with h5py.File(save_file) as f:
feat = cnn_patch_feature(net, transformer, im, bbox[i, :, :])
f[data_split][i, :] = feat
print '[{:d}] {} [{:.3f} sec]'.format(i,
osp.split(im_files[i])[-1],
timer.toc())
def train(model, beam_searcher, train_set, valid_set, save_dir, lr,
display=100, starting=0, endding=20, validation=2000, life=10, logger=None):
"""
display: output training infomation every 'display' mini-batches
starting: the starting snapshots, > 0 when resuming training
endding: the least training snapshots
validation: evaluate on validation set every 'validation' mini-batches
life: increase of endding when finds better model
"""
train_func, _ = adam_optimizer(model, lr=lr)
print '... training'
logger = Logger(save_dir) if logger is None else logger
timer = Timer()
loss = 0
imb = starting * validation
best = -1
best_snapshot = -1
timer.tic()
while imb < endding*validation:
imb += 1
x = train_set.iterate_batch()
loss += train_func(*x)[0] / display
if imb % display == 0:
logger.info('snapshot={}, iter={}, loss={:.6f}, time={:.1f} sec'.format(imb/validation, imb, loss, timer.toc()))
timer.tic()
loss = 0
if imb % validation == 0:
saving_index = imb/validation
model.save_to_dir(save_dir, saving_index)
try:
scores = validate(beam_searcher, valid_set, logger)
if scores[3] > best:
best = scores[3]
best_snapshot = saving_index
endding = max(saving_index+life, endding)
logger.info(' ---- this Bleu-4 = [%.3f], best Bleu-4 = [%.3f], endding -> %d' % \
(scores[3], best, endding))
except OSError:
print '[Ops!! OS Error]'
logger.info('Training done, best snapshot is [%d]' % best_snapshot)
return best_snapshot
def HASE(b4, A_inverse, b_cov, C, N_con, DF):
with Timer() as t:
B13 = b_cov
B4 = b4
A1_B_constant = np.tensordot(A_inverse[:, :, 0:(N_con)],
B13,
axes=([2], [0]))
A1_B_nonconstant = np.einsum('ijk,il->ijl',
A_inverse[:, :, N_con:N_con + 1], B4)
A1_B_full = A1_B_constant + A1_B_nonconstant
BT_A1B_const = np.einsum('ij,lji->li', B13.T, A1_B_full[:,
0:(N_con), :])
BT_A1B_nonconst = np.einsum('ijk,ijk->ijk', B4[:, None, :],
A1_B_full[:, (N_con):N_con + 1, :])
BT_A1B_full = BT_A1B_const[:, None, :] + BT_A1B_nonconst
C_BTA1B = BT_A1B_full - C.reshape(1, -1)
C_BTA1B = np.abs(C_BTA1B)
a44_C_BTA1B = C_BTA1B * A_inverse[:, (N_con):N_con + 1,
(N_con):N_con + 1]
a44_C_BTA1B = np.sqrt((a44_C_BTA1B))
t_stat = np.sqrt(DF) * np.divide(A1_B_full[:, (N_con):N_con + 1, :],
a44_C_BTA1B)
SE = a44_C_BTA1B / np.sqrt(DF)
print "time to compute GWAS for {} phenotypes and {} SNPs .... {} sec".format(
b4.shape[1], A_inverse.shape[0], t.secs)
return t_stat, SE
def run(self, matrix, maxiter=30): # matrix shape(#doc, #word)
"""
Run the Gibbs sampler.
"""
n_docs, vocab_size = matrix.shape
self._initialize(matrix)
for it in range(maxiter):
timer = Timer()
timer.start()
print('--- iter '+str(it))
for m in range(n_docs):
# 在doc m下的第i個字 -- w (此處的w是td_matrix中，word的index)
for i, w in enumerate(word_indices(matrix[m, :])):
z = self.topics[(m, i)]
self.nmz[m, z] -= 1
self.nm[m] -= 1
self.nzw[z, w] -= 1
self.nz[z] -= 1
p_z = self._conditional_distribution(m, w)
z = sample_index(p_z)
self.nmz[m, z] += 1
self.nm[m] += 1
self.nzw[z, w] += 1
self.nz[z] += 1
self.topics[(m, i)] = z
timer.print_time()
print('--- end iter')
# FIXME: burn-in and lag!
yield self.phi_pzw()
def _HRZColumnsMap(self):
return {}
def _CollectInHomeDeviceID(self):
return [
"W634iMCwmSCcjQkltb7d38btv000%02d" % i
for i in [18, 6, 23, 1, 13, 12, 4, 5, 24, 14]
]
def _unnormalDataPrecess(self, a, b, c, d):
return a, b, c, d
class BXYMain(MainModel):
def dataReader(self, startDate, endDate, HRZID, Code):
return XK(startDate, endDate, HRZID, Code)
def genePsudoWeather(self, stageData, weatherIndex, pLength):
return [-1]
if __name__ == '__main__':
# 声明当前使用的时钟对象
Clock = Timer('2018-02-26 00:00:00', 15)
# 声明当前使用的模型对象
Model = CNNModel
BXY_1 = BXYMain(True, '2017-12-15 00:00:00', Clock, 30, Model, [
u"瞬时流量", u'气象站室外温度', u"气象站室外湿度", u"气象站室外风速", u'气象站室外光照', u"回水压力",
u'供水流量', u"燃气温度"
], [u'一次回温度'], [1], False, [1, 2, 3, 4], [0])
testVariable = BXY_1.main()
def test_mvtec(test_set, rebuilder, transform, save_dir, threshold_seg_dict, configs):
_t = Timer()
cost_time = list()
if not os.path.exists(os.path.join(save_dir, 'ROC_curve')):
os.mkdir(os.path.join(save_dir, 'ROC_curve'))
for item in test_set.test_dict:
s_map_list = list()
s_map_good_list=list()
item_dict = test_set.test_dict[item]
if not os.path.exists(os.path.join(save_dir, item)):
os.mkdir(os.path.join(save_dir, item))
os.mkdir(os.path.join(save_dir, item, 'ori'))
os.mkdir(os.path.join(save_dir, item, 'gen'))
os.mkdir(os.path.join(save_dir, item, 'mask'))
#os.mkdir(os.path.join(save_dir, item))
for type in item_dict:
if not os.path.exists(os.path.join(save_dir, item, 'ori', type)):
os.mkdir(os.path.join(save_dir, item, 'ori', type))
if not os.path.exists(os.path.join(save_dir, item, 'gen', type)):
os.mkdir(os.path.join(save_dir, item, 'gen', type))
if not os.path.exists(os.path.join(save_dir, item, 'mask', type)):
os.mkdir(os.path.join(save_dir, item, 'mask', type))
_time = list()
img_list = item_dict[type]
for path in img_list:
image = cv2.imread(path, cv2.IMREAD_COLOR)
ori_h, ori_w, _ = image.shape
_t.tic()
ori_img, input_tensor = transform(image)
out = rebuilder.inference(input_tensor)
re_img = out.transpose((1, 2, 0))
s_map = ssim_seg_mvtec(ori_img, re_img, configs, win_size=3, gaussian_weights=True)
if threshold_seg_dict: # dict is not empty
mask = seg_mask_mvtec(s_map, threshold_seg_dict[item],configs)
else:
mask = seg_mask_mvtec(s_map, 64, configs)
inference_time = _t.toc()
img_id = path.split('.')[0][-3:]
cv2.imwrite(os.path.join(save_dir, item, 'ori', type, '{}.png'.format(img_id)), ori_img)
cv2.imwrite(os.path.join(save_dir, item, 'gen', type, '{}.png'.format(img_id)), re_img)
cv2.imwrite(os.path.join(save_dir, item, 'mask', type, '{}.png'.format(img_id)), mask)
_time.append(inference_time)
if type != 'good':
s_map_bad=s_map.reshape(-1,1)
s_map_list.append(s_map_bad)
else:
s_map_good = s_map.reshape(-1, 1)
s_map_good_list.append(s_map_good)
cost_time += _time
mean_time = np.array(_time).mean()
print('Evaluate: Item:{}; Type:{}; Mean time:{:.1f}ms'.format(item, type, mean_time*1000))
_t.clear()
torch.save(s_map_list, os.path.join(save_dir, item) + '/s_map.pth')
torch.save(s_map_good_list, os.path.join(save_dir, item) + '/s_map_good.pth')
# calculate mean time
cost_time = np.array(cost_time)
cost_time = np.sort(cost_time)
num = cost_time.shape[0]
num90 = int(num*0.9)
cost_time = cost_time[0:num90]
mean_time = np.mean(cost_time)
print('Mean_time: {:.1f}ms'.format(mean_time*1000))
# evaluate results
print('Evaluating...')
test_set.eval(save_dir)
def train():
args = get_arguments()
print '*' * 10 + ' args ' + '*' * 10
print args
print '*' * 26
if not os.path.exists(args.save_dir):
os.mkdir(args.save_dir)
if args.data_provider == "coco":
#pass
coco_provider_train = COCO_detection_train(args.root_dir,
args.json_file_train)
coco_provider_val = COCO_detection_train(args.root_dir,
args.json_file_val)
data_provider_train = MultiDataProvider([coco_provider_train],
crop_size=args.crop_size,
min_ratio=min_ratio,
max_ratio=max_ratio,
hor_flip=hor_flip)
data_provider_val = MultiDataProvider([coco_provider_val],
crop_size=args.crop_size,
min_ratio=1,
max_ratio=1.1,
hor_flip=False)
else:
raise RuntimeError, 'unknown data provider type'
solver_param = SolverParameter()
solver_param.type = args.type
solver_param.base_lr = args.learning_rate
solver_param.lr_policy = args.lr_policy
solver_param.gamma = args.gamma
solver_param.stepsize = args.step_size
solver_param.gpu_list = args.gpus_list
solver_param.exclude_scope = args.exclude_scope
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
sess = tf.Session(config=config)
solver = Solver(solver_param, sess)
total_size = args.batch_size * solver.num_gpus
print('total size for single forward: ', total_size)
with tf.name_scope('input'):
inputs = {
'images':
tf.placeholder(tf.float32,
shape=(total_size, args.crop_size, args.crop_size,
3),
name='input_images')
}
label_gt = {
'labels':
tf.placeholder(dtype=tf.int32,
shape=(total_size, int(args.crop_size * scale),
int(args.crop_size * scale)),
name='input_label')
}
global_step = tf.train.get_or_create_global_step()
nas_refine_net = nas_refine_model(num_cls=args.num_classes,
is_training=True,
ohem=args.ohem,
mining_ratio=args.mining_ratio)
train_op, total_loss, metric = solver.deploy(
model_fn=nas_refine_net.infer,
loss_fn=nas_refine_net.loss_fn,
eval_fn=nas_refine_net.eval_metric,
inputs=inputs,
targets=label_gt,
total_size=total_size,
regularize=True)
miou = metric['miou']
tf.summary.scalar('miou', miou)
tf.summary.scalar('total_loss', total_loss)
tf.summary.scalar('lr', solver.learning_rate)
### init
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
### restore
flag = STATUS.NO_INIT
variables_to_restore = None
if args.init_weights != None:
flag = STATUS.FINTTUNE
# if not os.path.exists(args.init_weights):
# raise RuntimeError, '{} does not exist for finetuning'.format(args.init_weights)
init_weights = args.init_weights
variables_to_restore = solver.get_variables_finetune()
if args.solver_state != None:
flag = STATUS.CONTINUE
if os.path.isdir(args.solver_state):
solver_state = tf.train.latest_checkpoint(args.solver_state)
else:
# if not os.path.exists(args.solver_state):
# raise RuntimeError, '{} does not exist for continue training'.format(args.solver_state, flag)
solver_state = args.solver_state
variables_to_restore = solver.get_variables_continue_training()
if flag == STATUS.FINTTUNE:
loader = tf.train.Saver(var_list=variables_to_restore)
loader.restore(sess, init_weights)
print('{} loaded'.format(init_weights))
elif flag == STATUS.CONTINUE:
loader = tf.train.Saver(var_list=variables_to_restore)
loader.restore(sess, solver_state)
print('{} loaded'.format(solver_state))
all_summaries = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(args.save_dir)
saver = tf.train.Saver()
label_sz = int(args.crop_size * scale)
#timer = Timer()
for step in range(1, args.num_steps):
#print 'step: ', step
#data_blob = np.zeros((total_size, args.crop_size, args.crop_size, 3), dtype=np.float32)
#label_blob = np.zeros((total_size, label_sz, label_sz), dtype=np.int32)
data_blob = np.zeros((0, args.crop_size, args.crop_size, 3),
dtype=np.float32)
label_blob = np.zeros((0, label_sz, label_sz), dtype=np.int32)
timer = Timer()
timer.tic()
for cur_id in range(solver.num_gpus):
#start = int(cur_id * args.batch_size)
#end = int((cur_id + 1) * args.batch_size)
while True:
images, labels = data_provider_train.get_batch(args.batch_size)
#data_blob[start:end] = images
labels_resize = np.zeros((labels.shape[0], label_sz, label_sz),
dtype=np.int32)
for n in range(labels.shape[0]):
labels_resize[n] = cv2.resize(
labels[n], (label_sz, label_sz),
interpolation=cv2.INTER_NEAREST)
#label_blob[start:end] = labels_resize
if np.any(labels_resize >= 0):
data_blob = np.concatenate((data_blob, images), axis=0)
label_blob = np.concatenate((label_blob, labels_resize),
axis=0)
break
assert label_blob.shape[0] == total_size
assert data_blob.shape[0] == total_size
### run training op
_, losses_value, _, summary, global_step_val = sess.run(
[train_op, total_loss, metric['op'], all_summaries, global_step],
feed_dict={
inputs['images']: data_blob,
label_gt['labels']: label_blob
})
summary_writer.add_summary(summary, global_step=global_step_val)
### show
if step % args.show_steps == 0:
t1 = timer.toc()
print(
'step: {}, lr: {}, loss_value: {}, miou: {}, time: {} / per iter'
.format(global_step_val, sess.run(solver.learning_rate),
losses_value, miou.eval(session=sess), t1))
#time = timer.toc()
## save
if step % args.save_step == 0:
save(saver, sess, args.save_dir, step=global_step_val)
### test
if step % args.val_steps == 0:
test_loss = 0
print('#' * 5 + ' testing ' + '#' * 5)
for kk in range(test_iter):
for cur_id in range(solver.num_gpus):
start = int(cur_id * args.batch_size)
end = int((cur_id + 1) * args.batch_size)
while True:
images, labels = data_provider_val.get_batch(
args.batch_size)
data_blob[start:end] = images
labels_resize = np.zeros(
(labels.shape[0], label_sz, label_sz),
dtype=np.int32)
for n in range(labels.shape[0]):
labels_resize[n] = cv2.resize(
labels[n], (label_sz, label_sz),
interpolation=cv2.INTER_NEAREST)
label_blob[start:end] = labels_resize
if np.any(labels_resize >= 0):
break
losses_value = sess.run([total_loss],
feed_dict={
inputs['images']: data_blob,
label_gt['labels']: label_blob
})
test_loss += losses_value[0]
print('global_step_val: {}, test loss: {}'.format(
global_step_val,
float(test_loss) / test_iter))
print('#' * 19)
def test_mvtec(test_set, rebuilder, transform, save_dir, threshold_seg_dict, val_index):
_t = Timer()
cost_time = list()
threshold_dict = dict()
if not os.path.exists(os.path.join(save_dir, 'ROC_curve')):
os.mkdir(os.path.join(save_dir, 'ROC_curve'))
for item in test_set.test_dict:
threshold_list = list()
item_dict = test_set.test_dict[item]
if not os.path.exists(os.path.join(save_dir, item)):
os.mkdir(os.path.join(save_dir, item))
os.mkdir(os.path.join(save_dir, item, 'ori'))
os.mkdir(os.path.join(save_dir, item, 'gen'))
os.mkdir(os.path.join(save_dir, item, 'mask'))
for type in item_dict:
if not os.path.exists(os.path.join(save_dir, item, 'ori', type)):
os.mkdir(os.path.join(save_dir, item, 'ori', type))
if not os.path.exists(os.path.join(save_dir, item, 'gen', type)):
os.mkdir(os.path.join(save_dir, item, 'gen', type))
if not os.path.exists(os.path.join(save_dir, item, 'mask', type)):
os.mkdir(os.path.join(save_dir, item, 'mask', type))
_time = list()
img_list = item_dict[type]
for path in img_list:
image = cv2.imread(path, cv2.IMREAD_COLOR)
ori_h, ori_w, _ = image.shape
_t.tic()
ori_img, input_tensor = transform(image)
out = rebuilder.inference(input_tensor)
re_img = out.transpose((1, 2, 0))
s_map = ssim_seg(ori_img, re_img, win_size=11, gaussian_weights=True)
s_map = cv2.resize(s_map, (ori_w, ori_h))
if val_index == 1:
mask = seg_mask(s_map, threshold=threshold_seg_dict[item])
elif val_index == 0:
mask = seg_mask(s_map, threshold=threshold_seg_dict)
else:
raise Exception("Invalid val_index")
inference_time = _t.toc()
img_id = path.split('.')[0][-3:]
cv2.imwrite(os.path.join(save_dir, item, 'ori', type, '{}.png'.format(img_id)), ori_img)
cv2.imwrite(os.path.join(save_dir, item, 'gen', type, '{}.png'.format(img_id)), re_img)
cv2.imwrite(os.path.join(save_dir, item, 'mask', type, '{}.png'.format(img_id)), mask)
_time.append(inference_time)
if type != 'good':
threshold_list.append(s_map)
else:
pass
cost_time += _time
mean_time = np.array(_time).mean()
print('Evaluate: Item:{}; Type:{}; Mean time:{:.1f}ms'.format(item, type, mean_time*1000))
_t.clear()
threshold_dict[item] = threshold_list
# calculate mean time
cost_time = np.array(cost_time)
cost_time = np.sort(cost_time)
num = cost_time.shape[0]
num90 = int(num*0.9)
cost_time = cost_time[0:num90]
mean_time = np.mean(cost_time)
print('Mean_time: {:.1f}ms'.format(mean_time*1000))
# evaluate results
print('Evaluating...')
test_set.eval(save_dir,threshold_dict)
def run(cfg: config.EvolutionConfig) -> None:
toolbox = create_toolbox(cfg)
population = toolbox.population(n=cfg.population_size)
fitnesses = [toolbox.evaluate(el) for el in population]
for ind, fit in zip(population, fitnesses):
ind.fitness.values = fit
CXPB, MXPB = 0.3, 0.3
child_per_parent = 2
i = 0
with Timer() as timer, SolutionTracer(
filename=
f"Evolutionary_CXPB_{CXPB}_MXPB_TS_{TOURNAMENT_SIZE}_{MXPB}_CPP_{child_per_parent}_I_{cfg.max_iterations}_PS_{cfg.population_size}",
max_repetitions=cfg.max_iterations,
id=cfg.id,
clues=cfg.clues) as solution_tracer:
while cfg.max_iterations > i:
i += 1
offspring = population
# Clone the selected individuals
offspring = list(map(toolbox.clone, child_per_parent * offspring))
for child1, child2 in zip(offspring[::2], offspring[1::2]):
if random.random() < CXPB:
# if random.random() < 0.5:
toolbox.mate_score(child1, child2)
# else:
# toolbox.mate_r(child1, child2)
del child1.fitness.values
del child2.fitness.values
for mutant in offspring:
if random.random() < MXPB:
if random.random() < 0.8:
toolbox.mutate_swap_many(mutant)
else:
toolbox.mutate_swap_one(mutant)
del mutant.fitness.values
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = map(toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
best_offspring = []
if child_per_parent > 1:
for begin, end in zip(
range(0,
len(offspring) - child_per_parent + 1,
child_per_parent),
range(child_per_parent,
len(offspring) + 1, child_per_parent)):
best_offspring.append(
sorted(offspring[begin:end],
key=lambda x: x.fitness.values[0])[0])
else:
best_offspring = offspring
population = choose_unique(best_offspring)
population = toolbox.select_t(population, cfg.population_size)
# saving and checking stats
best = tools.selBest(population, 1)[0]
solution_tracer.update(best, timer.elapsed)
if best.fitness.values[0] == 0:
break
if i % 100 == 0:
print(i, best.fitness)
print(best.sudoku)
best = tools.selBest(population, 1)[0]
print(best.fitness)
print(best.sudoku)
# with Timer('Explicit inversion'):
# A_inv = np.linalg.inv(A)
# x = np.dot(A_inv, -s) # fun fact: this can be also written as A_inv @ (-s)
# scipy's LU factorization
# with Timer('LU solver'):
# LU, piv = sl.lu_factor(A, overwrite_a=False, check_finite=False)
# x = sl.lu_solve((LU, piv), -s)
# scipy version of linalg.solve
# with Timer('scipy.linalg.solve'):
# x = sl.solve(A, -s, assume_a='sym', overwrite_a=False, overwrite_b=True, check_finite=False)
# low-level LAPACK solver
lapack_gesv = sl.get_lapack_funcs('gesv', (A, -s))
with Timer('Low-level LAPACK solver'):
_, _, x, _ = lapack_gesv(A, -s)
# LDL decomposition - rather slow and pretty numerically unstable
# with Timer('LDL decomposition'):
# lu, d, perm = sl.ldl(A, overwrite_a=True, check_finite=False)
# L = lu[perm]
# z = sl.solve(L, -s, overwrite_a=True, check_finite=False)
# y = sl.solve(d, z, overwrite_a=True, check_finite=False)
# x = sl.solve(d @ L.T, y, overwrite_a=True, check_finite=False)
# plot the sites, colored with their value of x
plt.figure(figsize=(8, 8))
sc = plt.scatter(positions[:, 0], positions[:, 1], c=x, cmap='viridis')
plt.gca().set_aspect('equal')
plt.colorbar(sc, fraction=0.046, pad=0.035)
def discreteness_constraint(x):
return (x - 0.5)*(x - 0.5) - 0.25
def constr_jac(x):
return 2 * np.identity(n) * x - 1.
def constr_hess(x, v):
return 2 * np.identity(n) * v
# linear constraint matrix
C = np.ones((1, n), dtype=np.float32)
# C[1:] = np.identity(n, dtype=np.float32)
# lb = np.zeros(n+1, dtype=np.float32)
# lb[0] = m
# ub = np.ones(n+1, dtype=np.float32)
# ub[0] = m
with Timer('minimizer'):
res = so.minimize(energy, x0=np.ones_like(s), jac=jac, hess=hess, method='trust-constr',
constraints=[so.LinearConstraint(C, m, m),
so.NonlinearConstraint(discreteness_constraint, 0, 1e-8)])
x = res.x
# x = np.round(res.x)
print(f"Final energy: {energy(x)}")
print(x)
print(np.sum(x))
# plot the sites, colored with their value of x
plt.figure(figsize=(8, 8))
sc = plt.scatter(positions[:, 0], positions[:, 1], c=x, cmap='viridis')
plt.gca().set_aspect('equal')
plt.colorbar(sc, fraction=0.046, pad=0.035)
plt.show()
with open(cfg_file, "r") as f:
configs = json.load(f)
start_epoch = configs['op']['start_epoch']
max_epoch = configs['op']['max_epoch']
learning_rate = configs['op']['learning_rate']
decay_rate = configs['op']['decay_rate']
epoch_steps = configs['op']['epoch_steps']
snapshot = configs['op']['snapshot']
batch_size = configs['db']['batch_size']
loader_threads = configs['db']['loader_threads']
save_dir = configs['system']['save_dir']
# init Timer
if not os.path.exists(save_dir):
os.mkdir(save_dir)
_t = Timer()
# create log file
log = Log(args.log_dir, args.cfg)
log.wr_cfg(configs)
# load data set
training_set = load_data_set_from_factory(configs, 'train')
print('Data set: {} has been loaded'.format(configs['db']['name']))
# load model
trainer = load_training_model_from_factory(configs, ngpu=args.ngpu)
if configs['system']['resume']:
trainer.load_params(configs['system']['resume_path'])
print('Model: {} has been loaded'.format(configs['model']['name']))
def fit( self, data,
iterations=1000,
batch_size=64,
k=2,
out_dir='./out',
out_iter=5 ):
#some helpful things
ones = np.ones( ( batch_size, 1 ) ).astype( 'float32' )
minus_ones = ones * -1.
timer = Timer( iterations )
output_pattern = out_dir + '/{:0' + str( len( str( iterations ) ) ) + 'd}.png' #erghh if it is stupid but it works it is not stupid
clear = '\r '
progress = '{} | D( loss:\t{:0.2f}, diff:\t{:0.2f}, norm:\t{:0.2f}, ; G( loss:\t{:0.2f} )'
#get some noise:
out_samples = self.make_some_noise()
distance_samples = self.make_some_noise( 10 )
print( distance( distance_samples ) )
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# print( distance( np.clip( ( self.generate( distance_samples ) + 1 ) * 127.5, 0, 255 ) ) )
io.imsave( out_dir + '/real_samples.png',
image_grid( np.clip( ( data[ : 64 ] + 1 ) * 127.5, 0, 255 ) ), # make sure we have valid values
plugin='pil' )
for i in range( iterations ):
timer.start_step()
#train discriminator
self.make_trainable( True )
for j in range( k ):
real_data = data[ np.random.choice( data.shape[0], batch_size, replace=False ), : ]
fake_data = self.generate( self.make_some_noise( batch_size ) )
epsilon = np.random.random( batch_size )
interpolation = ( real_data.T * epsilon ).T + ( fake_data.T * ( 1 - epsilon ) ).T
d_loss, d_diff, d_norm = self.dis_trainer.train_on_batch( [ real_data, fake_data, interpolation ], [ ones ] * 2 )
##something messed up
# for l in self.dis_trainer.layers:
# weights = l.get_weights()
# replace = False
# for j, w in enumerate( weights ):
# if np.isnan( w ).any():
# weights[ j ] = np.nan_to_num( w )
# replace = True
# if replace:
# l.set_weights( weights )
# if replace:
# print('\nfucking NaN man')
#trian generator
self.make_trainable( False )
g_loss = self.gan.train_on_batch( self.make_some_noise( batch_size ), minus_ones )
##something messed up
# for l in self.gan.layers:
# weights = l.get_weights()
# replace = False
# for j, w in enumerate( weights ):
# if np.isnan( w ).any():
# weights[ j ] = np.nan_to_num( w )
# replace = True
# if replace:
# l.set_weights( weights )
# if replace:
# print('\nfucking NaN man')
if np.isnan( d_loss ):
for j,l in enumerate( self.gan.layers):
for k, w in enumerate( l.get_weights() ):
w = np.nan_to_num( w )
print( '{}/{}: {}/{}'.format( j, k, np.min( w ), np.max( w ) ) )
if i % out_iter == 0:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# print( distance( np.clip( ( self.generate( distance_samples ) + 1 ) * 127.5, 0, 255 ) ) )
io.imsave( output_pattern.format( i ),
image_grid( np.clip( ( self.generate( out_samples ) + 1 ) * 127.5, 0, 255 ) ), # make sure we have valid values
plugin='pil' )
timer.stop_step()
#progess reporting
sys.stdout.write( clear )
sys.stdout.flush()
sys.stdout.write( '\r' + progress.format( timer.out_str(), d_loss, d_diff, d_norm, g_loss ) )
sys.stdout.flush()
def start(self):
self.timer = Timer.Timer(self.config.flight_fix_scan_interval, self.do)
self.timer.start()
