def request_argv(args, pre_api=None):
'''
命令行参数解析成dict类型，并执行。参数里可以有$a/b/c 类型变量
:param args: cmd参数组成的dict
:return: 执行的结果
'''
param_dic = {}
if '-p' in args.keys():
v = args['-p']
if re.match("\{('\w+'\s*:\s*'?\$?\w+'?)(?:\s*,\s*'\w+':\s*'?\$?\w+)*'?\}", v):
'''
预先加载的接口中，有$变量引用的，先替换变量，被应用的变量，需是常量，或者在引用之前被计算
'''
reg = '(\$(?:\*|[\w/]+))'
var_list = re.findall(reg, v)
# getStudentInfo.do -p {'studentId':$studentId}
for va in var_list:
rand = utils.random_select(utils.find_by_path(pre_api, va[1:]))
v.replace(va, str(rand))
param_dic['-p'] = v
# -p {userId:1, loginId:123@qq, ...} 转化成dict
if re.match("\{('\w+':\s*'?[\w@]+'?)(?:\s*,\s*'\w+':\s*'?[\w@]+'?)*}", v):
param_dic = eval(v)
# ready to request
cfg = Apis()
api_dic = cfg.find_api(args['name'])
api_dic['params'].update(param_dic)
newp = Api(api_dic, True if '-v' in args.keys() else False)
return newp.request_and_find(args['-j'] if '-j' in args.keys() else None)
def __init__(self, api_info, verbose=False):
self.counter = 0
self.verbose = verbose
if isinstance(api_info, str):
self.api = json.loads(api_info, encoding='utf-8', object_pairs_hook=OrderedDict)
elif isinstance(api_info, dict):
self.api = api_info
# 对于dict对象api来说，结构对应于params的一个api，对于其第一层key-value，如果有$xxx/xxx，或者@getApi ... 这类值，首先处理
# $之后的语句，认为是一个路径，通过该路径可以找到json结构中的某个值，并引用。
# @之后，认为是一个接口依赖，会首先请求，并把相应值挂在到json结构该处
for k, v in api_info.items():
if isinstance(v, str):
if '$' in v:
# 匹配$a/b/c形式字符串
reg = '(\$(?:\*|[\w/]+))'
var_list = re.findall(reg, v)
for va in var_list:
rand = utils.random_select(utils.find_by_path(self.api, va[1:]))
v = v.replace(va, str(rand) if rand else '')
api_info[k] = v
# 匹配@apiName形式字符串
if isinstance(v, str) and v.startswith('@'):
api_info[k] = self.request_cmd(v[1:])
saver = tf.train.Saver(max_to_keep=10, var_list=var_list)
with multi_gpu.create_session() as sess:
sess.run(tf.global_variables_initializer())
ckpt_file = tf.train.latest_checkpoint(result_path)
begin_epoch = 1
if ckpt_file is not None:
print('Restoring model from {}...'.format(ckpt_file))
begin_epoch = int(ckpt_file.split('.')[-4]) + 1
saver.restore(sess, ckpt_file)
for epoch in range(begin_epoch, epoches + 1):
if epoch % anneal_lr_freq == 0:
learning_rate *= anneal_lr_rate
x_train_source, t_train_source = utils.random_select(
x_train.reshape(n_y, -1, n_x), t_train)
x_train_source = np.tile(x_train_source, (n_y, 1))
t_train_source = np.tile(t_train_source, (n_y, 1))
x_train_tmp, t_train_tmp = utils.shuffle(
np.concatenate((x_train.reshape(
-1, n_x, 1), x_train_source.reshape(-1, n_x, 1)),
axis=2),
np.concatenate((t_train.reshape(
-1, n_code, 1), t_train_source.reshape(-1, n_code, 1)),
axis=2))
x_train_shuffle = x_train_tmp[:, :, 0].reshape(-1, n_x)
t_train_shuffle = t_train_tmp[:, :, 0].reshape(-1, n_code)
x_train_source = x_train_tmp[:, :, 1].reshape(-1, n_x)
t_train_source = t_train_tmp[:, :, 1].reshape(-1, n_code)
# x_train = np.reshape(x_train, [-1, n_xl, n_xl, 1])
lower_bounds = []
def params(self):
'''
"@if $type==1: '@getNewAccount.do {'type':3,'classId':$classId}' else"
'''
ps = self.api['params']
for k in ps.keys():
v = ps[k]
if isinstance(v, dict) or isinstance(v, list):
v = json.dumps(v)
if not isinstance(v, str):
continue
if '%' in v:
v = v % (self.counter)
if re.match(r'\{[\u4e00-\u9fa5\w]+(,[\u4e00-\u9fa5\w]+)*\}', v):
v = utils.random_select(v)
# 1. 处理$变量
if '$' in v:
reg = '(\$(?:\*|[\w/]+))'
var_list = re.findall(reg, v)
for va in var_list:
if va == '$*':
del ps[k]
# print(ps)
v = v.replace(va, json.dumps({k: v for k, v in ps.items() if v is not None}))
ps[k] = v
else:
rand = utils.random_select(utils.find_by_path(self.api, va[1:]))
if va[1:] in ps.keys():
ps[va[1:]] = rand if rand else ''
v = v.replace(va, str(rand))
# 1. 可执行代码块
if re.findall('!\{\{.+?\}\}', v):
if re.findall('!\{\{.+?\}\}', v) or re.findall('!<<.+?>>', v):
v = utils.eval_all(v)
# @后面的代码
if v.startswith('@'):
while isinstance(v, str) and v.startswith('@'):
cmd = v[1:]
if cmd.startswith('if '):
if_start = cmd.find(':', cmd.index('if')) + 1
if_end = len(cmd)
else_sentence = None
if cmd.find('else') != -1:
if_end = cmd.find('else', if_start)
else_start = cmd.find('else:')
else_sentence = cmd[else_start + len('else:'):].strip()
if_cmd = cmd[len('if'):if_start - 1].strip()
if_sentence = cmd[if_start:if_end].strip()
if eval(if_cmd):
v = if_sentence
else:
v = else_sentence
else:
v = self.request_cmd(cmd)
# 准备废弃的语法
if isinstance(v, str) and '@' in v:
at_list = re.findall(r'@<<.+?>>', v)
for al in at_list:
v = v.replace(al, self.request_cmd(al[3:-2]))
ps[k] = v
return {k: v for k, v in ps.items() if v is not None}
def get_code():
for i in params:
print(i.name, i.get_shape())
var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope='decoder') + \
tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope='encoder')
saver = tf.train.Saver(max_to_keep=10, var_list=var_list)
with multi_gpu.create_session() as sess:
sess.run(tf.global_variables_initializer())
ckpt_file = tf.train.latest_checkpoint(result_path)
begin_epoch = 1
if ckpt_file is not None:
print('Restoring model from {}...'.format(ckpt_file))
begin_epoch = int(ckpt_file.split('.')[-4]) + 1
saver.restore(sess, ckpt_file)
for epoch in range(begin_epoch, epoches + 1):
if epoch % anneal_lr_freq == 0:
learning_rate *= anneal_lr_rate
x_train_source, t_train_source = utils.random_select(x_train.reshape(n_y, -1, n_x), t_train)
x_train_source = np.tile(x_train_source,(n_y,1))
t_train_source = np.tile(t_train_source,(n_y,1))
x_train_tmp, t_train_tmp = utils.shuffle(
np.concatenate((x_train.reshape(-1, n_x, 1), x_train_source.reshape(-1, n_x, 1)), axis=2),
np.concatenate((t_train.reshape(-1, n_code, 1), t_train_source.reshape(-1, n_code, 1)), axis=2))
x_train_shuffle = x_train_tmp[:, :, 0].reshape(-1, n_x)
t_train_shuffle = t_train_tmp[:, :, 0].reshape(-1, n_code)
x_train_source = x_train_tmp[:, :, 1].reshape(-1, n_x)
t_train_source = t_train_tmp[:, :, 1].reshape(-1, n_code)
# x_train = np.reshape(x_train, [-1, n_xl, n_xl, 1])
lower_bounds = []
tv_losses = []
time_train = -time.time()
for t in range(train_iters):
iter = t + 1
x_batch = x_train_shuffle[t * train_batch_size:(t + 1) * train_batch_size]
x_batch_bin = sess.run(x_bin, feed_dict={x_orig: x_batch})
t_batch = t_train_shuffle[t * train_batch_size:(t + 1) * train_batch_size]
x_batch_source = x_train_source[t * train_batch_size:(t + 1) * train_batch_size]
x_batch_source_bin = sess.run(x_bin, feed_dict={x_orig:x_batch_source})
t_batch_source = t_train_source[t * train_batch_size:(t + 1) * train_batch_size]
_, lb , tv = sess.run([infer, lower_bound , tv_loss],
feed_dict={x_orig: x_batch, x: x_batch_bin, code: t_batch,
x_source: x_batch_source_bin, code_source: t_batch_source,
learning_rate_ph: learning_rate, is_training: True})
lower_bounds.append(lb)
tv_losses.append(tv)
if iter % print_freq == 0:
print('Epoch={} Iter={} ({:.3f}s/iter): '
'Lower Bound={} , Tv loss={}'.
format(epoch, iter,
(time.time() + time_train) / print_freq,
np.mean(lower_bounds) , np.mean(tv_losses)))
lower_bounds = []
tv_losses = []
if iter % test_freq == 0:
time_test = -time.time()
t_batch = t_test[:gen_size]
gen_images = sess.run(eval_x_gen,
feed_dict={is_training: False,
code: t_batch})
name = "gen_{}/iwae_hccr.epoch.{}.iter.{}.png".format(n_y, epoch, iter)
name = os.path.join(result_path, name)
utils.save_image_collections(gen_images, name, shape=(test_ny, display_each_character),
scale_each=True)
# train reconstruction
x_batch = x_train_recon[:recon_size].reshape(-1, n_x)
x_batch_bin = sess.run(x_bin, feed_dict={x_orig: x_batch})
t_batch = t_train_recon[:recon_size]
eval_zs, recon_images = \
sess.run([eval_z_gen.tensor, eval_x_recon],
feed_dict={x: x_batch_bin, is_training: False, code: t_batch})
name = "train_recon_{}/iwae_hccr.epoch.{}.iter.{}.png".format(n_y,
epoch, iter)
name = os.path.join(
result_path, name)
utils.save_contrast_image_collections(x_batch.reshape(-1, n_xl, n_xl, n_channels), recon_images,
name, shape=(test_ny, display_each_character * 2),
scale_each=True)
# # train interpolation
x_batch_bin = sess.run(x_bin, feed_dict={x_orig: x_train_interp})
t_batch = t_train_interp
eval_zs, _ = \
sess.run([eval_z_gen.tensor, eval_x_recon],
feed_dict={x: x_batch_bin, is_training: False, code: t_batch})
epsilon = np.linspace(0, 1, display_each_character)
eval_zs_interp = np.array(
[eps * eval_zs[0, 2 * i, :] + (1 - eps) * eval_zs[0, 2 * i + 1, :] for i in range(test_ny) for eps
in epsilon]).reshape(1, -1, n_z)
t_batch = np.tile([t_batch[2 * i, :] for i in range(test_ny)], (1, display_each_character)).reshape(-1,
n_code)
recon_images = \
sess.run(eval_x_interp, feed_dict={interp_z: eval_zs_interp, is_training: False, code: t_batch})
name = "interp_{}/iwae_hccr.epoch.{}.iter.{}.png".format(n_y, epoch, iter)
name = os.path.join(result_path, name)
utils.save_image_collections(recon_images, name, shape=(test_ny, display_each_character),
scale_each=True)
# test reconstruction
x_batch = x_test[:recon_size].reshape(-1, n_x)
x_batch_bin = sess.run(x_bin, feed_dict={x_orig: x_batch})
t_batch = t_test[:recon_size]
eval_zs, recon_images = \
sess.run([eval_z_gen.tensor, eval_x_recon],
feed_dict={x: x_batch_bin, is_training: False, code: t_batch})
name = "test_recon_{}/iwae_hccr.epoch.{}.iter.{}.png".format(n_y,
epoch, iter)
name = os.path.join(
result_path, name)
utils.save_contrast_image_collections(x_batch.reshape(-1, n_xl, n_xl, n_channels), recon_images,
name, shape=(test_ny, display_each_character * 2),
scale_each=True)
# one-shot generation
x_batch = x_oneshot_test.reshape(-1, n_x) # display_number*nxl*nxl*nchannel
x_batch_bin = sess.run(x_bin, feed_dict={x_orig: x_batch})
t_batch = t_oneshot_test
display_x_oneshot = np.zeros((display_each_character, test_ny+ 1, n_xl, n_xl, n_channels))
eval_zs_oneshot = sess.run(eval_z_oneshot.tensor,
feed_dict={x: x_batch_bin, is_training: False, code: t_batch})
# print (np.shape(eval_zs_oneshot)) #test_ny*nz
for i in range(display_each_character):
display_x_oneshot[i, 0, :, :, :] = x_batch[i, :].reshape(-1, n_xl, n_xl, n_channels)
tmp_z = np.zeros((1, test_ny, n_z))
for j in range(test_ny):
# print (np.shape(tmp_z) ,np.shape(eval_zs_oneshot))
tmp_z[0, j, :] = eval_zs_oneshot[0, i, :]
# _, eval_x_oneshot = decoder({'z': oneshot_z}, tf_ny, code, is_training)
#print tmp_z.shape , t_oneshot_gen_test.shape
tmp_x = sess.run(eval_x_oneshot,
feed_dict={oneshot_z: tmp_z, tf_ny: test_ny, code: t_oneshot_gen_test,
is_training: False})
# print (np.shape(tmp_x))
display_x_oneshot[i, 1:, :, :, :] = tmp_x
display_x_oneshot = np.reshape(display_x_oneshot, (-1, n_xl, n_xl, n_channels))
##TODO
display_x_oneshot = (display_x_oneshot > 0.5).astype(np.float32)
name = "oneshot_{}/iwae_hccr.epoch.{}.iter.{}.png".format(n_y,
epoch, iter)
name = os.path.join(
result_path, name)
utils.save_image_collections(display_x_oneshot,
name, shape=(display_each_character, test_ny + 1),
scale_each=True)
# disentangle
t_batch = t_test[:recon_size]
z_each = np.random.normal(size=(display_each_character, n_z))
# print (z_each.shape)
z_batch = np.zeros((test_ny, display_each_character, n_z))
# print (z_batch.shape)
for i in range(test_ny):
z_batch[i, :, :] = z_each
z_batch = np.reshape(z_batch, (-1, n_z))
eval_disentange_x = \
sess.run(disentangle_x,
feed_dict={disentange_z: z_batch, is_training: False, code: t_batch})
name = "disentangle_{}/iwae_hccr.epoch.{}.iter.{}.png".format(n_y,
epoch, iter)
name = os.path.join(
result_path, name)
utils.save_image_collections(eval_disentange_x,
name, shape=(test_ny, display_each_character),
scale_each=True)
time_test += time.time()
if iter % save_freq == 0:
save_path = "iwae.epoch.{}.iter.{}.ckpt".format(epoch, iter)
save_path = os.path.join(result_path, save_path)
saver.save(sess, save_path)
if iter % print_freq == 0:
time_train = -time.time()
