def get_reynolds_stress_errors(data, avg_vel_by_loc, avg_vel_by_loc_w, lower, start=0, end=0.18, step=0.01): if type(avg_vel_by_loc) is str: avg_vel_by_loc = tls.read_json(avg_vel_by_loc) if type(avg_vel_by_loc_w) is str: avg_vel_by_loc_w = tls.read_json(avg_vel_by_loc_w) index = -1 if lower: index = 0 def param_f(elem, i): position = str(tls.group_by_location(elem, i)).replace("-0.0", "0.0") return (-elem.velocity()[i][0] - sorted(avg_vel_by_loc[position], key=lambda x: x[0])[index][0]) * \ (-sorted(avg_vel_by_loc_w[position], key=lambda x: x[0])[index][0] + elem.velocity()[i][1]) return tls.group_parameter( data, lambda t, i: tls.group_by_height(t, i, start, end, step), param_f)
def get_acc(vel): sl = ft.Scene("/home/ron/Desktop/Alexey/the_dataset/traj_" + vel + "_low.h5") sh = ft.Scene("/home/ron/Desktop/Alexey/the_dataset/traj_" + vel + "_high.h5") acc_low = group_avarage_acc(sl, group_by_x_n_z) acc_to_save = {} for key in acc_low.keys(): acc_to_save[key] = [acc_low[key][0].tolist(), acc_low[key][1]] tls.save_as_json(acc_to_save, "accel_by_x_and_z_" + vel + "_lower") print("Lower done") acc_high = group_avarage_acc(sh, group_by_x_n_z) acc_to_save = {} for key in acc_high.keys(): acc_to_save[key] = [acc_high[key][0].tolist(), acc_high[key][1]] tls.save_as_json(acc_to_save, "accel_by_x_and_z_" + vel + "_higher") print("Higher done") m = tls.merge_dict( tls.read_json("accel_by_x_and_z_" + vel + "_higher"), tls.read_json("accel_by_x_and_z_" + vel + "_lower"), lambda a, b: [((np.array(a[0]) * a[1] + np.array(b[0]) * b[1]) / (a[1] + b[1])).tolist(), a[1] + b[1]]) tls.save_as_json(m, "accel_by_x_and_z_" + vel) print("DONE")
def get_dispersive_stress(data, avg_vel_by_loc, avg_vel_by_h, start=0, end=0.18, step=0.01): if type(avg_vel_by_loc) is str: avg_vel_by_loc = tls.read_json(avg_vel_by_loc) if type(avg_vel_by_h) is str: avg_vel_by_h = tls.read_json(avg_vel_by_h) def param_f(elem, i): position = tls.group_by_location(elem, i) pos_str = str(position).replace("-0.0", "0.0") height = str(abs(position[1])) + " - " + str(abs(position[1]) + step) if height not in avg_vel_by_h.keys(): height = str(abs(position[1]) - step) + " - " + str( abs(position[1])) return (-avg_vel_by_loc[pos_str][0][0] + avg_vel_by_h[height][0][0]) * \ (avg_vel_by_loc[pos_str][0][1] - avg_vel_by_h[height][0][1]) return tls.group_parameter( data, lambda t, i: tls.group_by_height(t, i, start, end, step), param_f)
def sum_all_acc(vel, area=0.05 * 0.01, mult=mass, only_corner=True, version="2"): acc = tls.read_json("accel" + version + "_by_x_and_z_" + vel) acc_err = tls.read_json("Statistics/acc_nb_xz_mult_avgs_" + vel) total = {} count = {} num = {} err = {} Cd = {} Cd_g = {} for key in acc.keys(): if only_corner and -float(key.split(", ")[0].replace('(', "")) <= 0.12: continue if key not in acc_err.keys(): print key + " is missing" continue h = float(key.split(", ")[1].replace(")", "")) if h in total.keys(): total[h] += np.array(acc[key][0]) * mult err[h][0] += sorted(acc_err[key], key=lambda a: a[0])[0][0] * mult err[h][1] += sorted(acc_err[key], key=lambda a: a[0])[-1][0] * mult count[h] += acc[key][1] num[h] += 1.0 else: total[h] = np.array(acc[key][0]) * mult count[h] = acc[key][1] err[h] = [0, 0] err[h][0] = sorted(acc_err[key], key=lambda a: a[0])[0][0] * mult err[h][1] = sorted(acc_err[key], key=lambda a: a[0])[-1][0] * mult num[h] = 1.0 tmp = raupach.get_drag_raupach(vel)["v"] for key in total.keys(): total[key] = [total[key] / num[key], count[key]] Cd_g[key] = (-total[key][0][0] * 2) / (air_density * (float(vel)**2) * area), count[key] err[key][0] = abs((((err[key][0] / num[key]) * 2) / (air_density * (float(vel)**2) * area)) - Cd_g[key][0]) err[key][1] = abs((((err[key][1] / num[key]) * 2) / (air_density * (float(vel)**2) * area)) - Cd_g[key][0]) try: Cd[key] = (-total[key][0][0] * 2) / ( air_density * (tmp[round(key * 100.0) - 0.5]**2) * area), count[key] except KeyError: pass return total, Cd, Cd_g, err
def quiver_velocity(vel): acc = tls.read_json("raupach_data/avg_vel_by_loc_" + vel) fig, ax = pplot.subplots() matx = np.zeros((18, 18)) matcount = np.zeros((18, 18)) matz = np.zeros((18, 18)) for key in acc.keys(): x = int(float(key.split(", ")[0].replace("(", "")) * 100.0) z = int(float(key.split(", ")[1]) * 100.0) if x > 0: x *= -1 matx[z, -x] += -acc[key][0][0] * acc[key][1] matz[z, -x] += acc[key][0][1] * acc[key][1] matcount[z, -x] += acc[key][1] Y = np.zeros((18, 18)) X = np.zeros((18, 18)) for i in range(len(Y)): for j in range(len(Y)): Y[i, j] = i / 10.0 X[i, j] = j / 10.0 if matcount[i, j] != 0: matx[i, j] /= matcount[i, j] matz[i, j] /= matcount[i, j] ax.quiver(X, Y, matx, matz, units="inches", scale=1) ax.set_xlabel(r"x/H") ax.set_ylabel(r"z/H") return fig, ax
def draw_quiver(vel): acc = tls.read_json("accel_by_x_and_z_" + vel) fig, ax = pplot.subplots() matx = np.zeros((18, 18)) matz = np.zeros((18, 18)) for key in acc.keys(): if acc[key][1] < 10000: continue x = int(float(key.split(", ")[0].replace("(", "")) * 100) if x > 0: continue z = int(float(key.split(", ")[1].replace(")", "")) * 100) matx[z, -x] = math.sqrt(abs(-acc[key][0][0])) * np.sign(-acc[key][0][0]) matz[z, -x] = math.sqrt(abs(acc[key][0][1])) * np.sign(acc[key][0][1]) Y = np.zeros((18, 18)) X = np.zeros((18, 18)) for i in xrange(len(Y)): for j in xrange(len(Y)): Y[i, j] = i / 10.0 X[i, j] = j / 10.0 ax.quiver(X, Y, matx, matz, units="inches", scale=8) ax.set_xlabel(r"x/H") ax.set_ylabel(r"z/H") return fig, ax
def read_problem(in_path): raw = tools.read_json(in_path) start = convex.ConvexPolygon(raw["start"]) end = convex.ConvexPolygon(raw["end"]) obstacles = [convex.ConvexPolygon(ver_i) for ver_i in raw["collision"]] pol_envir = collision.PolygonEnvir(obstacles) return Problem(start, end, pol_envir)
def calc_vel_and_drag_from_data_Cd(data, area=0.0005, acc=minimum_acc): ret = {} if type(data) is str: data = tls.read_json(data) x_vel = [] for key in sorted(data.keys()): if data[key][1] < acc or key == "no group": continue key_splat = key.split(' - ') height = (float(key_splat[0]) + float(key_splat[1])) / 2 x_vel.append([-data[key][0][0], height]) ret["x_velocities"] = x_vel drag_list = [] for elem in x_vel: drag_list.append([estimate_drag_Cd(elem[0], area), elem[1]]) ret["drag_list"] = drag_list ret["drag"] = reduce(lambda a, b: a + b[0], drag_list, 0) return ret
class TestApprove(unittest.TestCase): # 初始化 def setUp(self) -> None: # 获取session对象 self.session = requests.session() # 获取ApiCommon对象 self.common = ApiCommon(self.session) # 获取ApiApprove对象 self.approve = self.common.get_api_approve() # 结束 def tearDown(self) -> None: # 关闭session self.session.close() # 1、认证接口 测试方法 @parameterized.expand(read_json("approve.json", "approve")) def test01_approve(self, realname, card_id, response_code, status_code, expect_msg): # 调用登录 self.common.get_api_reg_login().api_login("13600001111", "test12345") # 调用 认证接口 r = self.approve.api_approve(realname, card_id) log.info("认证结果为:{}".format(r.json())) print("认证结果为:", r.json()) # 断言 common_assert(self, r, response_code, status_code, expect_msg) # 2、认证查询接口 测试方法 @parameterized.expand(read_json("approve.json", "get_approve")) def test02_get_approve(self, login_name, response_code, phone_4): # 调用登录 self.common.get_api_reg_login().api_login(login_name, "test12345") # 调用查询认证接口 r = self.approve.api_get_approve() log.info("认证查询结果为:{}".format(r.json())) # 断言 common_assert(self, r, response_code=response_code, status_code=None) self.assertEqual(response_code, r.status_code) if phone_4: # 扩展 断言手机号 尾号 phone = r.json().get("phone") print("phone:", phone) self.assertIn(phone_4, phone)
def get_error(file_name, h, mult=1.0): data = tls.read_json(file_name) for key in data.keys(): splat = key.split(" - ") firstNum = float(splat[0]) secondNum = float(splat[1]) if h > firstNum and h < secondNum: rel_array = data[key] rel_array.sort(key=lambda a: a[0]) return np.array([rel_array[0][0], rel_array[-1][0]]) * mult
def get_average_error(value_group, error_group, limit=10000): value_data = read_json(value_group) error_data = read_json(error_group) avg = 0.0 count = 0.0 for key in value_data.keys(): if key in error_data.keys(): value = value_data[key] if value[1] > limit: acc = -value[0][0] err_orderd = list(map(lambda a: a[0], error_data[key])) err_orderd.sort() avg += (abs(err_orderd[0] - acc) + abs(err_orderd[-1] - acc)) / 2.0 count += 1 return avg / count
def get_error_cd_modified(file_name, h): data = tls.read_json(file_name) for key in data.keys(): splat = key.split(" - ") firstNum = float(splat[0]) secondNum = float(splat[1]) if h > firstNum and h < secondNum: rel_array = data[key] rel_array.sort(key=lambda a: a[0]) return np.array([(estimate_drag_Cd(rel_array[0][0], 0.0005)), (estimate_drag_Cd(rel_array[-1][0], 0.0005))])
def get_reynolds_stress(data, avg_vel_by_loc, start=0, end=0.18, step=0.01): if type(avg_vel_by_loc) is str: avg_vel_by_loc = tls.read_json(avg_vel_by_loc) def param_f(elem, i): position = str(tls.group_by_location(elem, i)).replace("-0.0", "0.0") return (avg_vel_by_loc[position][0][0] - elem.velocity()[i][0]) * \ (-avg_vel_by_loc[position][0][1] + elem.velocity()[i][1]) return tls.group_parameter( data, lambda t, i: tls.group_by_height(t, i, start, end, step), param_f)
def restore(self, filename, model_and_loss, include_params="*", exclude_params=()): # ----------------------------------------------------------------------------------------- # Make sure file exists # ----------------------------------------------------------------------------------------- if not os.path.isfile(filename): logging.info("Could not find checkpoint file '%s'!" % filename) quit() # check which file version is latest no_extension = filename.split('.')[0] statistics_filename = no_extension + ".json" statistics = tools.read_json(statistics_filename) logging.info('JSON: ' + str(statistics)) shadow_is_latest = statistics['shadow'] logging.info('Shadow is latest: ' + str(shadow_is_latest)) # ----------------------------------------------------------------------------------------- # Load checkpoint from file including the state_dict # ----------------------------------------------------------------------------------------- if shadow_is_latest: checkpoint_with_state = torch.load(no_extension + '_shadow' + '.ckpt') else: checkpoint_with_state = torch.load(filename) # ----------------------------------------------------------------------------------------- # Load filtered state dictionary # ----------------------------------------------------------------------------------------- state_dict = checkpoint_with_state[self._model_key] restore_keys = tools.filter_list_of_strings( state_dict.keys(), include=include_params, exclude=exclude_params) state_dict = {key: value for key, value in state_dict.items() if key in restore_keys} self._load_state_dict_into_module(state_dict, model_and_loss) # logging.info(" Restore keys:") # for key in restore_keys: # logging.info(" %s" % key) # ----------------------------------------------------------------------------------------- # Get checkpoint statistics without the state dict # ----------------------------------------------------------------------------------------- checkpoint_stats = { key: value for key, value in checkpoint_with_state.items() if key != self._model_key } logging.info('STATS ' + str(checkpoint_stats)) return checkpoint_stats, filename
def update(self): if not os.path.exists(self.tlsfile): print("瓦片集配置文件:" + self.tlsfile + "不存在!") return self.tlsconf = tools.read_json(self.tlsfile) self.tlsdir, _ = os.path.split(self.tlsfile) # 各种配置路径 self.source = self.tlsdir + "/" + self.tlsconf["source"] self.dest = self.tlsdir + "/" + self.tlsconf["dest"] # 遍历所有配置 if os.path.isdir(self.source): if not os.path.isdir(self.dest): os.makedirs(self.dest) for tlconf in self.tlsconf["tilesets"]: self.gen_tileset(tlconf)
def configure_checkpoint_saver(args, model_and_loss): with logger.LoggingBlock("Checkpoint", emph=True): checkpoint_saver = CheckpointSaver() checkpoint_stats = None if args.checkpoint is None: logging.info("No checkpoint given.") logging.info("Starting from scratch with random initialization.") elif os.path.isfile(args.checkpoint): logging.info("Loading checkpoint file %s" % args.checkpoint) checkpoint_stats, filename = checkpoint_saver.restore( filename=args.checkpoint, model_and_loss=model_and_loss, include_params=args.checkpoint_include_params, exclude_params=args.checkpoint_exclude_params) # load epoch number no_extension = filename.split('.')[0] statistics_filename = no_extension + ".json" statistics = tools.read_json(statistics_filename) args.start_epoch = statistics['epoch'] + 1 elif os.path.isdir(args.checkpoint): if args.checkpoint_mode in ["resume_from_best"]: logging.info("Loading best checkpoint in %s" % args.checkpoint) checkpoint_stats, filename = checkpoint_saver.restore_best( directory=args.checkpoint, model_and_loss=model_and_loss, include_params=args.checkpoint_include_params, exclude_params=args.checkpoint_exclude_params) elif args.checkpoint_mode in ["resume_from_latest"]: logging.info("Loading latest checkpoint in %s" % args.checkpoint) checkpoint_stats, filename = checkpoint_saver.restore_latest( directory=args.checkpoint, model_and_loss=model_and_loss, include_params=args.checkpoint_include_params, exclude_params=args.checkpoint_exclude_params) else: logging.info("Unknown checkpoint_restore '%s' given!" % args.checkpoint_restore) quit() else: logging.info("Could not find checkpoint file or directory '%s'. Starting with random initialization." % args.checkpoint) return checkpoint_saver, checkpoint_stats
def filter_video(video_name, json_files, video_path): os.makedirs(video_path + '/json_tmp/', exist_ok=True) cap = cv2.VideoCapture(video_path + '/' + video_name) fps = int(math.ceil(cap.get(5))) #FPS width = int(cap.get(3)) # WIDTH height = int(cap.get(4)) # HEIGHT font = cv2.FONT_HERSHEY_SIMPLEX #Creates a font fourcc = cv2.VideoWriter_fourcc(*'mp4v') out_debug = cv2.VideoWriter(video_path + '/debug.mp4', fourcc, fps, (width, height)) i = 0 point_x, point_y = width / 2, height / 2 while (cap.isOpened() and i < len(json_files)): ret, frame = cap.read() if ret: kps = read_json(video_path + '/openpose/json/' + json_files[i]) if len(kps) != 0: #get closer skeleton of the previous one json_path = video_path + '/json_tmp/' + json_files[i].split( '_')[2] + '.json' try: point_x, point_y, idx = get_skeleton( kps, json_path, point_x, point_y) except Exception as e: print(e) drawed_frame = draw(kps[idx], frame) drawed_frame = np.uint8(drawed_frame) cv2.putText(drawed_frame, json_files[i].split('/')[-1], (10, 50), font, 1, (0, 0, 255), 2, cv2.LINE_AA) #Draw the text out_debug.write(drawed_frame) i = i + 1 ## RM NON FILTER DIR cmd = "rm -rf " + video_path + '/openpose/json/' os.system(cmd) ## MV FILTERED FILES TO OPENPOSE DIR cmd = "mv " + video_path + '/json_tmp ' + video_path + '/openpose/json' os.system(cmd)
def save_latest(self, directory, model_and_loss, stats_dict, store_as_best=False): # ----------------------------------------------------------------------------------------- # Make sure directory exists # ----------------------------------------------------------------------------------------- tools.ensure_dir(directory) # check previous latest file version latest_statistics_filename = os.path.join( directory, self._prefix + self._latest_postfix + ".json") if os.path.isfile(latest_statistics_filename): statistics = tools.read_json(latest_statistics_filename) shadow_is_latest = statistics['shadow'] else: shadow_is_latest = True stats_dict['shadow'] = not shadow_is_latest # ----------------------------------------------------------------------------------------- # Save # ----------------------------------------------------------------------------------------- save_dict = dict(stats_dict) save_dict[self._model_key] = model_and_loss.state_dict() if shadow_is_latest: latest_checkpoint_filename = os.path.join( directory, self._prefix + self._latest_postfix + self._extension) else: latest_checkpoint_filename = os.path.join( directory, self._prefix + self._latest_postfix + '_shadow' + self._extension) torch.save(save_dict, latest_checkpoint_filename) tools.write_json(data_dict=stats_dict, filename=latest_statistics_filename) # ----------------------------------------------------------------------------------------- # Possibly store as best # ----------------------------------------------------------------------------------------- if store_as_best: best_checkpoint_filename = os.path.join( directory, self._prefix + self._best_postfix + self._extension) best_statistics_filename = os.path.join( directory, self._prefix + self._best_postfix + ".json") logging.info("Saved checkpoint as best model..") shutil.copyfile(latest_checkpoint_filename, best_checkpoint_filename) shutil.copyfile(latest_statistics_filename, best_statistics_filename)
def update(self): if not os.path.exists(self.headfile): print("头像配置文件:" + self.headfile + "不存在!") return self.headconf = tools.read_json(self.headfile) self.mdsdir, _ = os.path.split(self.headfile) self.source = self.mdsdir + "/" + self.headconf["source"] self.dest = self.mdsdir + "/" + self.headconf["dest"] if not os.path.exists(self.dest): os.makedirs(self.dest) # 遍历所有头像配置 for headc in self.headconf["heads"]: if headc["flipX"]: srcimg = Image.open(self.source + "/" + headc["image"]) srcimg = srcimg.transpose(Image.FLIP_LEFT_RIGHT) srcimg.save(self.dest + "/" + headc["head"]) else: shutil.copyfile(self.source + "/" + headc["image"], self.dest + "/" + headc["head"])
import tensorflow as tf import logging import time import numpy as np from datetime import timedelta from tqdm import tqdm from Model.Transformer import transformer_model, create_attention_mask_from_input_mask from tools import read_json, save_json from Optimization import create_optimizer logging.basicConfig(level=logging.INFO) events = ['EquityFreeze', 'EquityRepurchase', 'EquityUnderweight', 'EquityOverweight', 'EquityPledge'] cache = read_json('./tfrecord/ner_tag.json') tag_map = {} for k in cache.keys(): tag_map[cache[k]] = k events_fields = {'EquityFreeze': ['EquityHolder', 'FrozeShares', 'LegalInstitution', 'TotalHoldingShares', 'TotalHoldingRatio', 'StartDate', 'EndDate', 'UnfrozeDate'], 'EquityRepurchase': ['CompanyName', 'HighestTradingPrice', 'LowestTradingPrice',
def get_std_h(h, vel): data = read_json(std_root + str(vel)) relevant = str( filter(lambda a: abs(h - float(a)) < 0.0001, data.keys())[0]) print relevant return sqrt(data[relevant][0] / (data[relevant][1] - 1))
class TestTrust(unittest.TestCase): # 初始化 def setUp(self) -> None: # 获取session对象 self.session = requests.session() # 获取ApiCommon对象 self.common = ApiCommon(self.session) # 调用登录接口 self.common.get_api_reg_login().api_login("13600001111", "test12345") # 获取ApiTrust对象 self.trust = self.common.get_api_trust() # 结束 def tearDown(self) -> None: # 关闭session self.session.close() # 开户接口测试方法 @parameterized.expand(read_json("trust.json", "trust")) def test01_trust(self, expect_msg): log.info("开户参数化 expect_msc的值为:{}".format(expect_msg)) # 1、调用开户接口 r = self.trust.api_trust() log.info("开户结果:{}".format(r.json())) # 2、提取html数据 result = html_parser(r) log.info("提取开户三方请求数据结果:{}".format(result)) # 3、请求三方开户 r = self.session.post(result[0], data=result[1]) log.info("三方开户请求结果:{}".format(r.text)) # 4. 断言 self.assertEqual(expect_msg, r.text) # 充值验证码接口测试方法 @parameterized.expand(read_json("trust.json", "trust_code")) def test02_trust_code(self, random, response_code): r = self.trust.api_trust_code(random) print("r:", r.text) log.info("充值验证码返回响应状态码为:{}".format(r.status_code)) common_assert(self, r, response_code=response_code, status_code=None) # 充值接口测试方法 @parameterized.expand(read_json("trust.json", "trust_recharge")) def test03_trust_recharge(self, amount, valicode, expect_msg): # 充值验证码 self.trust.api_trust_code(12312313) # 1. 调用充值接口 r = self.trust.api_trust_recharge(amount, valicode) log.info("充值接口返回结果:{}".format(r.json())) if valicode != 8888: common_assert(self, r, status_code=100, description=expect_msg) else: # 2. 提取html result = html_parser(r) log.info("提取充值三方请求数据结果:{}".format(result)) # 3. 调用三方充值 r = self.session.post(result[0], data=result[1]) log.info("三方充值结果:{}".format(r.text)) # 4. 断言 print(r.text) self.assertEqual(expect_msg, r.text)
def initialize(self, data): self.read(data) self.num_trial = 0 self.response = read_json("static/style.json") self.style = read_json("static/style.json") print('initialize' + '-' * 30 + f'\n{self}')
def list_subcategories(): """ Return the subcategories from subcategories.json """ global LIST_SUBCATEGORIES LIST_SUBCATEGORIES = tools.read_json('subcategories.json') return LIST_SUBCATEGORIES
# import tools # from tools import * from tools import read_json, generate_string, get_slise # from lesson10 import generate_string import os print(__name__) filename = "Homeworks/data.json" result = read_json(filename) print(result) res = generate_string(10, 3) res = get_slise([1, 2, 3, 4, 5], 10, 3) print(res) folder_name = "Homeworks" files = [file for file in os.listdir(folder_name) if ".json" in file] # files = [file for file in os.listdir(folder_name) if os.path.isfile(os.path.join(folder_name, file))] print(files) for file in files: res = read_json(os.path.join(folder_name, file)) print(res)
class TestRegLogin(unittest.TestCase): # 初始化 def setUp(self) -> None: # 获取session对象 self.session = requests.session() logger.info("正在获取session对象:{}".format(self.session)) # 获取ApiRegLogin对象 self.api = ApiRegLogin(self.session) logger.info("正在获取ApiRegLogin对象:{}".format(self.api)) # 结束 def tearDown(self) -> None: logger.info("正在关闭session对象") # 关闭session对象 self.session.close() # 1.注册图片验证码 测试方法 @parameterized.expand(read_json("register_login.json","img_code_case")) def test01_img_code(self,random,expect_code): # 调用图片验证码接口 r = self.api.reg_img_code(random) # 断言 响应200 print("响应状态码:", r.status_code) try: self.assertEqual(expect_code, r.status_code) except Exception as e: logger.error(e) raise # 2. 注册 短信验证码 @parameterized.expand(read_json("register_login.json", "sms_code_case")) def test02_sms_code(self,phone,imgverifycode,type,expect_code,expect_status,description): # 1. 获取图片验证码 目的: 使用session对象自动记录cookie self.api.reg_img_code(random.random()) # 2. 获取短信验证码 r = self.api.reg_sms_code(phone,imgverifycode,type=type) print("获取短信验证码 结果为: ", r.json()) try: # 调用断言方法 common_assert(self, r,status_code=expect_code,status=expect_status, description=description) except Exception as e: logger.error(e) raise # 3. 注册 测试方法 @parameterized.expand(read_json("register_login.json", "reg_case")) def test03_reg(self,phone4,password,imgVerifyCode,phone_code,dy_server,invite_phone,expect_code,status,description): # 1. 获取图片验证码 self.api.reg_img_code(random.random()) # 2. 获取短信验证码 self.api.reg_sms_code(phone4, imgVerifyCode) # 3. 调用注册接口 r = self.api.reg(phone4, password, imgVerifyCode, phone_code, dy_server, invite_phone) print("响应状态码:", r.json()) logger.info("请求数据:{} 响应结果:{}".format( (phone4, password, imgVerifyCode, phone_code, dy_server, invite_phone, expect_code, status, description), r.json())) try: # 4. 断言注册信息 common_assert(self, r,status_code=expect_code,status=status,description=description) except Exception as e: logger.error(e) raise # 4. 登录 测试方法 @parameterized.expand(read_json("register_login.json", "login_case")) def test04_login(self,keywords, password, expect_code): r = self.api.login(keywords, password) logger.info("请求数据:{} 响应数据:{}".format((keywords, password, expect_code), r.json())) print("登录结果:", r.json()) if "error" in password: logger.info("锁定60验证...") r = self.api.login(keywords, password) logger.info("请求数据:{} 响应数据:{}".format((keywords, password, expect_code), r.json())) print("登陆结果:", r.json()) r = self.api.login(keywords, password) logger.info("请求数据:{} 响应数据:{}".format((keywords, password, expect_code), r.json())) print("登陆结果:", r.json()) time.sleep(60) r = self.api.login("13600001111", "q123456") logger.info("请求数据:{} 响应数据:{}".format((keywords, password, expect_code), r.json())) print("登陆结果:", r.json()) try: # 4. 断言登录信息 common_assert(self, r, status_code=expect_code) except Exception as e: logger.error(e) raise # 5. 是否登录 测试方法 @parameterized.expand(read_json("register_login.json", "islogin_case")) def test05_is_login(self,phone4,password,expect_code): # 1.调用登录 self.api.login(phone4, password) # 2.判断是否登录 r = self.api.islogin() print("登录查询结果:", r.json()) logger.info("请求数据:{} 执行结果:{}".format((phone4, password, expect_code), r.json())) try: common_assert(self, r, status=expect_code) except Exception as e: logger.error(e) raise
metrics=[CRF.sparse_accuracy]) model.summary() return model def train(self): history = self.model.fit_generator(self.generator.__iter__(), steps_per_epoch=len(self.generator), epochs=10) with open('trainHistoryDict.txt', 'wb') as file_pi: pickle.dump(history.history, file_pi) def predict(self): pass if __name__ == '__main__': ROOT_PATH = '/Users/ouhon/PycharmProjects/keras_nlp_tutorial/NER/lstm_crf/' path = ROOT_PATH + 'data/raw_data/dataset.jsonl' model_path = ROOT_PATH + 'lstm_crf.h5' data = read_jsonline(path) train_data = data[:int(len(data) * 0.8)] val_data = data[int(len(data) * 0.8):] token_dict = read_json(ROOT_PATH + 'data/raw_data/token2i.json') label_dict = read_json(ROOT_PATH + 'data/raw_data/label2i.json') i2label = {str(v): str(i) for i, v in label_dict.items()} generator = Data_generator(train_data, token_dict, label_dict) v_generator = Data_generator(val_data, token_dict, label_dict) lstm_crf = LstmCrf(token_dict, generator, v_generator) lstm_crf.train()
def __init__(self, config_file=None, exp_config=None, hh_config=None, prob_config=None): """ Initialise the experiment object. :param str config_file: Name of the configuration JSON file with the configuration dictionaries: exp_config, hh_config, and prob_config. If only the filename is provided, it is assumed that such a file is in the directory './exconf/'. Otherwise, the full path must be entered. The default value is None. :param dict exp_config: Configuration dictionary related to the experiment. Keys and default values are listed as follows: 'experiment_name': 'test', # Name of the experiment 'experiment_type': 'default', # Type: 'default', 'brute_force', 'basic_metaheuristics' 'heuristic_collection_file': 'default.txt', # Heuristic space located in /collections/ 'weights_dataset_file': None, # Weights or probability distribution of heuristic space 'use_parallel': True, # Run the experiment using a pool of processors 'parallel_pool_size': None, # Number of processors available, None = Default 'auto_collection_num_vals': 5 # Number of values for creating an automatic collection **NOTE 1:** 'experiment_type': 'default' or another name mean hyper-heuristic. **NOTE 2:** If the collection does not exist and it is not a reserved one ('default.txt', 'automatic.txt', 'basicmetaheuristics.txt', 'test_collection'), then an automatic heuristic space is generated with ``Operators.build_operators`` with 'auto_collection_num_vals' as ``num_vals`` and 'heuristic_collection_file' as ``filename``. **NOTE 3:** # 'weights_dataset_file' must be determined in a pre-processing step. For the 'default' heuristic space, it is provided 'operators_weights.json'. :param dict hh_config: Configuration dictionary related to the hyper-heuristic procedure. Keys and default values are listed as follows: 'cardinality': 3, # Maximum cardinality used for building metaheuristics 'num_agents': 30, # Population size employed by the metaheuristic 'num_iterations': 100, # Maximum number of iterations used by the metaheuristic 'num_replicas': 50, # Number of replicas for each metaheuristic implemented 'num_steps': 100, # * Number of steps that the hyper-heuristic performs 'max_temperature': 200, # * Initial temperature for HH-Simulated Annealing 'stagnation_percentage': 0.3, # * Percentage of stagnation used by the hyper-heuristic 'cooling_rate': 0.05 # * Cooling rate for HH-Simulated Annealing **NOTE 4:** Keys with * correspond to those that are only used when ``exp_config['experiment_type']`` is neither 'brute_force' or 'basic_metaheuristic'. :param dict prob_config: Configuration dictionary related to the problems to solve. Keys and default values are listed as follows: 'dimensions': [2, 5, 10, 20, 30, 40, 50], # List of dimensions for the problem domains 'functions': bf.__all__, # List of function names of the optimisation problems 'is_constrained': True # True if the problem domain is hard constrained :return: None. """ # If there is a configuration file if config_file: # Adjustments directory, filename = path.split(config_file) if directory == '': directory = './exconf/' # Default directory basename, extension = path.splitext(filename) if extension not in ['.json', '']: raise ExperimentError("Configuration file must be JSON") # Existence verification and load full_path = path.join(directory, basename + '.json') if path.isfile(full_path): all_configs = tl.read_json(full_path) # Load data from json file exp_config = all_configs['ex_config'] hh_config = all_configs['hh_config'] prob_config = all_configs['prob_config'] else: raise ExperimentError(f"File {full_path} does not exist!") else: if exp_config is None: exp_config = dict() if hh_config is None: hh_config = dict() if prob_config is None: prob_config = dict() # Load the default experiment configuration and compare it with exp_cfg self.exp_config = tl.check_fields( { 'experiment_name': 'test', 'experiment_type': 'default', # 'default' -> hh, 'brute_force', 'basic_metaheuristics' 'heuristic_collection_file': 'default.txt', 'weights_dataset_file': None, # 'operators_weights.json', 'use_parallel': True, 'parallel_pool_size': None, # Default 'auto_collection_num_vals': 5 }, exp_config) # Load the default hyper-heuristic configuration and compare it with hh_cfg self.hh_config = tl.check_fields( { 'cardinality': 3, 'num_agents': 30, 'num_iterations': 100, 'num_replicas': 50, 'num_steps': 100, 'max_temperature': 200, 'stagnation_percentage': 0.3, 'cooling_rate': 0.05 }, hh_config) # Load the default problem configuration and compare it with prob_config self.prob_config = tl.check_fields( { 'dimensions': [2, 5, *range(10, 50 + 1, 10)], 'functions': bf.__all__, 'is_constrained': True }, prob_config) # Check if there is a special case of function name: <choose_randomly> self.prob_config['functions'] = [ bf.__all__[hyp.np.random.randint(0, len(bf.__all__))] if fun == '<choose_randomly>' else fun for fun in self.prob_config['functions'] ] # Check if the heuristic collection exists if not path.isfile('./collections/' + self.exp_config['heuristic_collection_file']): # If the name is a reserved one. These files cannot be not created automatically if exp_config['heuristic_collection_file'] in [ 'default.txt', 'automatic.txt', 'basicmetaheuristics.txt', 'test_collection' ]: raise ExperimentError( 'This collection name is reserved and cannot be created automatically!' ) else: Operators.build_operators( Operators.obtain_operators( num_vals=exp_config['auto_collection_num_vals']), file_name=exp_config['heuristic_collection_file'].split( '.')[0]) self.exp_config['weights_dataset_file'] = None # Check if the weights dataset not exist or required if self.exp_config['weights_dataset_file'] and ( self.exp_config['experiment_type'] not in ['brute_force', 'basic_metaheuristics']): if path.isfile('collections/' + self.exp_config['weights_dataset_file']): self.weights_data = tl.read_json( 'collections/' + self.exp_config['weights_dataset_file']) else: raise ExperimentError( 'A valid weights_dataset_file must be provided in exp_config' ) else: self.weights_data = None
) ] history = self.model.fit_generator( self.generator.__iter__(), steps_per_epoch=len(generator), epochs=3, callbacks=callbacks_list, validation_data=self.v_generator.__iter__(), nb_val_samples=2000) def predict(self): pass if __name__ == '__main__': path = ROOT_PATH + 'data/toutiao/toutiao_train.csv' val_path = ROOT_PATH + 'data/toutiao/toutiao_test.csv' train_data = pd.read_csv(path) val_data = pd.read_csv(val_path) token_dict = read_json(ROOT_PATH + 'data/toutiao/token_dict.json') label_dict = read_json(ROOT_PATH + 'data/toutiao/label2i.json') train_D = [(x, y) for x, y in zip(train_data['x'], train_data['y'])][:10000] val_D = [(x, y) for x, y in zip(val_data['x'], val_data['y'])][:5000] generator = data_generator(train_D, token_dict, label_dict) v_generator = data_generator(val_D, token_dict, label_dict) print('build model') fast_text_model = FastText(token_dict, generator, v_generator) fast_text_model.train()
def plot_accl(vel): fig, ax = pplot.subplots() scale, sax = pplot.subplots() acc = tls.read_json("accel_by_x_and_z_" + vel) vals_raw = [] for item in acc.keys(): if acc[item][1] < 10000: continue x = int(float(item.split(", ")[0].replace("(", "")) * 100) if x > 0: continue vals_raw.append(-acc[item][0][0]) maximum = max(vals_raw) minimum = min(vals_raw) mat = np.zeros(shape=(18, 18, 3)).tolist() mat_s = np.zeros(shape=(1, 99, 3)).tolist() for key in acc.keys(): x = int(float(key.split(", ")[0].replace("(", "")) * 100) if x > 0: continue if acc[key][1] < 10000: continue z = int(float(key.split(", ")[1].replace(")", "")) * 100) val = ((-acc[key][0][0] - minimum) / (maximum - minimum)) mat[z][-x][0] = val mat[z][-x][1] = 0 mat[z][-x][2] = 1.0 - val ax.text(-x / 10.0 + 0.05, z / 10.0 + 0.05, round(-acc[key][0][0], 1), ha="center", va="center", color="w", size=7.8) for i in xrange(11): if i <= 5: mat[i][7] = [1, 1, 1] mat[i][15] = [1, 1, 1] val = 0.01 ind = 0 while val <= 1: mat_s[0][ind][0] = val mat_s[0][ind][1] = 0 mat_s[0][ind][2] = 1 - val ind += 1 val += 0.01 sax.imshow(mat_s, extent=[minimum, maximum, 0, 1], aspect='auto') sax.set_xticks( map(lambda a: round(a, 2), np.linspace(minimum, maximum, 12))) ax.imshow(mat, extent=[0, 1.8, 0, 1.8], origin="lower", aspect='auto') ax.set_xlabel(r"$x/H$") ax.set_ylabel(r"$z/H$") return fig, ax, scale, sax