def get_web_content(self, url, param, token_in_header=True): """ 使用指定参数,访问URL @param url: URL @param param: 参数对象 @param token_in_header: Token在Header中 @return: URL返回内容 """ # 文本编码 encoding = self.cfg['encoding'] header = self.get_header(token_in_header) # 添加Token参数 if not token_in_header and self.token is not None: param["token"] = self.token # 对象转JSON字符串 logcm.print_obj(param, "param") param_json = json.dumps(param, separators=(',', ':'), ensure_ascii=False) # 加密 encrypt_bytes = cryptcm.rsa_encrypt(self.pub_key_str, param_json.encode(encoding)) encrypt_str = encrypt_bytes.hex() # 做成URL web_url = "%s/%s?data=%s" % (self.cfg["server"], url, encrypt_str) logcm.print_obj(web_url, "web_url") # 打开Browser浏览器 content = self.client.open(web_url, "json") return content
def load_data(set_name, stock_code, start_date, end_date): """ :param set_name: 数据集名 :param stock_code: 股票代码,例如‘000001.SZ’ :param start_date: 回测开始日期,例如‘1991-1-30' :param end_date: 回测结束日期,例如‘2015-12-31’ :return: 函数返回其他函数的各参数序列 """ query = { "代码": stock_code, "日期": { "$gte": start_date, "$lte": end_date } } projection = { "_id": 0.0, "日期": 1.0, "代码": 1.0, "收盘价(元)": 1.0, "涨跌幅(%)": 1.0 } result = dbClient.find(set_name, query, projection) logcm.print_obj(result, "result") return result
def deskew(img): """ 抗扭矩计算 @:param img 图片 @return: """ # 计算得到的矩以一个字典的形式 m = cv2.moments(img) logcm.print_obj(m, "moments") if abs(m['mu02']) < 1e-2: return img.copy() # 移动矩阵 skew = m['mu11'] / m['mu02'] M = np.float32([[1, skew, -0.5 * SZ * skew], [0, 1, 0]]) # 仿射函数cv2.warpAffine()接受三个参数, # 需要变换的原始图像,移动矩阵M 以及变换的图像大小 # 移动矩阵M的格式:[[1, 0, tx],[0, 1, ty]] # (这个大小如果不和原始图像大小相同,那么函数会自动通过插值来调整像素间的关系) img = cv2.warpAffine(img, M, (SZ, SZ), flags=cv2.WARP_INVERSE_MAP | cv2.INTER_LINEAR) return img
def getColumns(self, table_name): """ 取得字段列表 :param table_name:表名 :return: 字段列表 """ logcm.print_info("Get columns of table: %s " % table_name) logcm.print_obj(table_name, "table_name")
def executemany(self, sql_command, param): """ 执行SQL命令 @param sql_command: SQL命令 @param param: SQL参数列表 @return: """ logcm.print_info("Execute Sql Command %s " % sql_command) logcm.print_obj(param, "param")
def fetchone(self, sql_command, param={}): """ 执行SQL命令 @param sql_command: SQL命令 @param param: SQL参数 @return: """ logcm.print_info("Fetch one with Sql Command %s " % sql_command) logcm.print_obj(param, "param")
def keys(self, pattern="*"): # 取得Key对应值 logcm.print_info('Redis keys: %s' % pattern) key_list = [] for key in self.rds.keys(pattern): # 转换成字符串 key_list.append(key.decode()) logcm.print_obj(key_list, "key_list", show_header=False) return key_list
def fetchmany(self, sql_command, param={}, pos=0): """ 执行SQL命令 @param sql_command: SQL命令 @param param: SQL参数 @param pos: 索引 @return: """ logcm.print_info("Fetch many with Sql Command %s " % sql_command) logcm.print_obj(param, "param") logcm.print_obj(pos, "pos")
def fetchone(self, sql_command, param={}): """ 执行SQL命令 @param sql_command: SQL命令 @param param: SQL参数 @return: """ self.cursor.execute(sql_command, param) # 获取一条记录 one_data = self.cursor.fetchone() logcm.print_obj(one_data, "one_data") return one_data
def fetchall(self, sql_command, param={}): """ 执行SQL命令 @param sql_command: SQL命令 @param param: SQL参数 @return: """ self.cursor.execute(sql_command, param) # 获取多条记录 all_data = self.cursor.fetchall() logcm.print_obj(all_data, "all_data") return all_data
def get_latest_buildno(self, job_name): """ 取得最近build版本号 @param job_name:JOB名称 @return: build版本号 """ job = self.get_job(job_name) lgb = job.get_last_good_build() logcm.print_obj(lgb, "lgb") return lgb.buildno
def Token(self): url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken' # corpid,corpsecret 为微信端获取 params = { 'corpid': self.corpid, 'corpsecret': self.corpsecret } url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken' r = requests.get(url=url, params=params) token = json.loads(r.text)['access_token'] logcm.print_obj(token, "token") return token
def fetchmany(self, sql_command, param={}, pos=0): """ 执行SQL命令 @param sql_command: SQL命令 @param param: SQL参数 @param pos: 索引 @return: """ self.cursor.execute(sql_command, param) # 获取多条记录 many_data = self.cursor.fetmany(pos) logcm.print_obj(many_data, "many_data") return many_data
def set_val(self, key, val): """ 设定Redis中指定Key的值 @param key: 指定Key @param val: 指定值 @return: 是否成功 """ # 取得Key对应值 logcm.print_info('Redis Set: %s --> %s' % (key, val)) result = self.rds.set(key, val) logcm.print_obj(result, "result", show_header=False) # 返回值 return result
def watch_job(self, job): """ 监视JOB @param job:JOB名称 @return: 无 """ if job: buildNo = job.get_last_buildnumber() + 1 consoleUrl = 'http://%s/jenkins/job/%s/%d/console' % ( self.cfg['host'], job.name, buildNo) logcm.print_obj(consoleUrl, "Console") print("<a class='linkBtn' href='%s' target='_blank'>查看控制台输出</a>" % consoleUrl)
def get_server_time(self): """ 取得服务器时间 :return: 服务器时间 """ cmd = 'date "+%Y/%m/%d %H:%M:%S" ' lines = self.exe_cmd(cmd) for line in lines: line = line.strip("\r\n") logcm.print_obj(line, "line") server_time = datetime.datetime.strptime(line, '%Y/%m/%d %H:%M:%S') logcm.print_obj(server_time, "date_time", show_header=False) return server_time return None
def get_last_build_param(self, job): """ 重复最近一次构建参数 @param job:JOB @return: 无 """ lgb = job.get_last_good_build() actions = lgb.get_actions() params = { pair['name']: pair['value'] for pair in actions["parameters"] } logcm.print_obj(params, "params", show_header=False) return params
def md5_encrypt(input_bytes): """ 对指定的字节列表进行MD5加密 @param input_bytes: 输入字节列表 @return: 加密后的字符串 """ # 创建md5对象 hl = hashlib.md5() hl.update(input_bytes) logcm.print_info("MD5 encrypt bytes : %s" % input_bytes) # 执行加密 result = hl.hexdigest() logcm.print_obj(result, "md5-encrypt-result", show_header=False) return result
def load_oracle_table(self, table_name): """ 按照配置读取指定数据表名,载入模块数据 :param table_name: 数据表名 :return:数据表字典对象 """ logcm.print_info("Loading table : %s ..." % table_name) # 加载DB数据表模块信息 # 建立和数据库系统的连接 dbClient = DbOracleClient("oracle", self.cfg_db) result = dbClient.getColumns(table_name) logcm.print_obj(result, "result", show_table=True) return result
def read_url(page_url, encoding): """ 通过URL得到网页内容 @param page_url: 请求的网页地址 @param encoding: 网页编码 @return: 网页文本 """ headers = {'User-Agent': random_agent()} req = urllib.request.Request(url=page_url, headers=headers) logcm.print_obj(req, 'req') response = urllib.request.urlopen(req) logcm.print_obj(response, 'response') html = response.read().decode(encoding, 'ignore') return html
def check_equal(obj1, obj2, title=""): """ 判断两个对象的值是否相等 @param obj1: 对象1 @param obj2: 对象2 @param title: 对象描述 @return: 是否OK """ ok = (obj1 == obj2) if ok: logcm.print_info("Assert Ok for %s" % title) else: logcm.print_info("Assert Equal Fail for %s" % title, color='bold', fg='red') logcm.print_obj(obj1, title + "-1", show_header=False) logcm.print_obj(obj2, title + "-2", show_header=False) return ok
def invoke_tag(self, job_name, tag_name, **kwargs): """ 启动JOB @param job_name:JOB名称ß @param tag_name:GitTag名 @return: 无 """ job = self.get_job(job_name) # start build the job params = {'GitVersion': tag_name} logcm.print_obj(params, "params", show_header=False) job.invoke(build_params=params) logcm.print_info("Jenkins Job %s is invoked." % job_name) self.watch_job(job)
def rsa_decrypt(prv_key_str, input_bytes): """ 按照指定的私钥字符串进行RSA解密 @param prv_key_str: 私钥字符串 @param input_bytes: 输入字节列表 @return: 解密后的字节列表 """ # 引入私钥字符串 prv_obj = rsa.importKey(prv_key_str) prv_obj = PKCS1_v1_5.new(prv_obj) # 执行解密 logcm.print_info("RSA decrypt bytes : %s" % input_bytes) result = prv_obj.decrypt(input_bytes, '') logcm.print_obj(result, "rsa-decrypt-result", show_header=False) return result
def rsa_encrypt(pub_key_str, input_bytes): """ 按照指定的公钥字符串进行RSA加密 @param pub_key_str: 公钥字符串 @param input_bytes: 输入字节列表 @return: 加密后的字节列表 """ # 引入公钥字符串 pub_obj = rsa.importKey(pub_key_str) pub_obj = PKCS1_v1_5.new(pub_obj) # 执行加密 logcm.print_info("RSA encrypt bytes : %s" % input_bytes) result = pub_obj.encrypt(input_bytes) logcm.print_obj(result, "rsa-encrypt-result", show_header=False) return result
def invoke(self, job_name, svn_url, task_no, **kwargs): """ 启动JOB @param job_name:JOB名称ß @param svn_url:SVN的URL路径 @param task_no:任务号 @return: 无 """ job = self.get_job(job_name) # start build the job params = {'url': svn_url, 'TASK_NO': task_no, 'svnPath': svn_url} logcm.print_obj(params, "params", show_header=False) job.invoke(build_params=params) logcm.print_info("Jenkins Job %s is invoked." % job_name) self.watch_job(job)
def xls_to_module(self, xls_path, sheet_name): """ 按照配置读取指定Exel,指定Sheet名,载入模块数据 :param xls_path: Excel文件路径 :param sheet_name: Sheet名 :return:模块数据字典对象 """ logcm.print_info("Loading xls : %s ..." % xls_path) # 加载Excel模块信息 cfg_mdl = self.cfg_xls['Module'] mdl_list = xlscm.load_excel_dict(xls_path, sheet_name, **cfg_mdl) # 模块信息为空判断 if mdl_list is None or len(mdl_list) == 0: logcm.print_info("Module Info is not set!", fg='red') sys.exit() if len(mdl_list) != 1: logcm.print_info("Module Info need only one!", fg='red') sys.exit() mdl_info = mdl_list[0]["Module"] mdl = Module(**mdl_info) # 加载接口信息 cfg_svc = self.cfg_xls['Services'] svc_list = xlscm.load_excel_dict(xls_path, sheet_name, **cfg_svc) for svc in svc_list: mdl.services.append(Service(svc)) # 加载Bean信息 cfg_bean = self.cfg_xls['Beans'] bean_list = xlscm.load_excel_dict(xls_path, sheet_name, **cfg_bean) for bean in bean_list: mdl.beans.append(Bean(bean)) # 加载Code信息 cfg_code = self.cfg_xls['Codes'] code_list = xlscm.load_excel_dict(xls_path, sheet_name, **cfg_code) for code in code_list: mdl.codes.append(Code(code)) logcm.print_obj(mdl, "mdl") return mdl
def get_header(self, token_in_header=True): """ 取得WEB访问Header @param token_in_header: Token在Header中 @return: Header对象 """ # Header对象 header = { "terminal": self.cfg["terminal"], "appType": self.cfg["appType"], "appVersionNo": self.cfg["appVersionNo"], "accessTerminal": self.cfg["accessTerminal"], "reqTime": datecm.get_now_num() } # 在Header中存放Token if token_in_header and self.token is not None: header["token"] = self.token logcm.print_obj(header, "header") return header
def invoke_task(self, job_name, task_no, **kwargs): """ 启动JOB @param job_name:JOB名称 @param task_no:任务号 @return: 无 """ # 取得JOB对象 job = self.get_job(job_name) # 最近build参数 params = self.get_last_build_param(job) # 替换任务号参数 params["TASK_NO"] = task_no logcm.print_obj(params, "params", show_header=False) job.invoke(build_params=params) logcm.print_info("Jenkins Job %s is invoked." % job_name) self.watch_job(job)
def get(self, key, convert=None): """ 取得Redis中指定Key的值 @param key: 指定Key @param convert: 指定转换方法 @return: Key对应值 """ # 取得Key对应值 logcm.print_info('Redis Get by key: %s' % key) try: result = self.rds.get(key) except Exception as e: logcm.print_info("Exception : %s" % e) result = None # 不存在时 if result is None: logcm.print_obj(result, "result", show_header=False) return None # 转换成字符串 result = result.decode() # 无转换时直接返回 if convert is None: logcm.print_obj(result, "result", show_header=False) return result # 执行转换 result = convert(result) logcm.print_obj(result, "result", show_header=False) return result
def load_diabetes(): """ Load the diabetes dataset @return: diabetes dataset by (data, target) """ # 文件路径 file_path_data = './cache/sk/sk_diabete_data.csv' file_path_target = './cache/sk/sk_diabete_target.csv' # 如果存在数据文件,则直接读取 if os.path.exists(file_path_data): # 读取文件 df_data = pd.read_csv(file_path_data) df_target = pd.read_csv(file_path_target) return df_data.as_matrix(), df_target.as_matrix() else: filecm.makedir(file_path_data, by_file=True) # 取得数据 diabetes = datasets.load_diabetes() logcm.print_obj(diabetes.DESCR, 'diabetes.DESCR') logcm.print_obj(diabetes.data, '样本原始数据集') logcm.print_obj(diabetes.target, '样本目标数据集') # 保存到文件 DataFrame(diabetes.data).to_csv(file_path_data, index=False) DataFrame(diabetes.target).to_csv(file_path_target, index=False) return diabetes.data, diabetes.target