def get_web_content(self, url, param, token_in_header=True):
"""
使用指定参数，访问URL
@param url: URL
@param param: 参数对象
@param token_in_header: Token在Header中
@return: URL返回内容
"""
# 文本编码
encoding = self.cfg['encoding']
header = self.get_header(token_in_header)
# 添加Token参数
if not token_in_header and self.token is not None:
param["token"] = self.token
# 对象转JSON字符串
logcm.print_obj(param, "param")
param_json = json.dumps(param,
separators=(',', ':'),
ensure_ascii=False)
# 加密
encrypt_bytes = cryptcm.rsa_encrypt(self.pub_key_str,
param_json.encode(encoding))
encrypt_str = encrypt_bytes.hex()
# 做成URL
web_url = "%s/%s?data=%s" % (self.cfg["server"], url, encrypt_str)
logcm.print_obj(web_url, "web_url")
# 打开Browser浏览器
content = self.client.open(web_url, "json")
return content
def load_data(set_name, stock_code, start_date, end_date):
"""
:param set_name: 数据集名
:param stock_code: 股票代码，例如‘000001.SZ’
:param start_date: 回测开始日期，例如‘1991-1-30'
:param end_date: 回测结束日期，例如‘2015-12-31’
:return: 函数返回其他函数的各参数序列
"""
query = {
"代码": stock_code,
"日期": {
"$gte": start_date,
"$lte": end_date
}
}
projection = {
"_id": 0.0,
"日期": 1.0,
"代码": 1.0,
"收盘价(元)": 1.0,
"涨跌幅(%)": 1.0
}
result = dbClient.find(set_name, query, projection)
logcm.print_obj(result, "result")
return result
def deskew(img):
"""
抗扭矩计算
@:param img 图片
@return:
"""
# 计算得到的矩以一个字典的形式
m = cv2.moments(img)
logcm.print_obj(m, "moments")
if abs(m['mu02']) < 1e-2:
return img.copy()
# 移动矩阵
skew = m['mu11'] / m['mu02']
M = np.float32([[1, skew, -0.5 * SZ * skew], [0, 1, 0]])
# 仿射函数cv2.warpAffine()接受三个参数，
# 需要变换的原始图像，移动矩阵M 以及变换的图像大小
# 移动矩阵M的格式：[[1, 0, tx],[0, 1, ty]]
# （这个大小如果不和原始图像大小相同，那么函数会自动通过插值来调整像素间的关系）
img = cv2.warpAffine(img,
M, (SZ, SZ),
flags=cv2.WARP_INVERSE_MAP | cv2.INTER_LINEAR)
return img
def getColumns(self, table_name):
"""
取得字段列表
:param table_name:表名
:return: 字段列表
"""
logcm.print_info("Get columns of table: %s " % table_name)
logcm.print_obj(table_name, "table_name")
def executemany(self, sql_command, param):
"""
执行SQL命令
@param sql_command: SQL命令
@param param: SQL参数列表
@return:
"""
logcm.print_info("Execute Sql Command %s " % sql_command)
logcm.print_obj(param, "param")
def fetchone(self, sql_command, param={}):
"""
执行SQL命令
@param sql_command: SQL命令
@param param: SQL参数
@return:
"""
logcm.print_info("Fetch one with Sql Command %s " % sql_command)
logcm.print_obj(param, "param")
def keys(self, pattern="*"):
# 取得Key对应值
logcm.print_info('Redis keys: %s' % pattern)
key_list = []
for key in self.rds.keys(pattern):
# 转换成字符串
key_list.append(key.decode())
logcm.print_obj(key_list, "key_list", show_header=False)
return key_list
def fetchmany(self, sql_command, param={}, pos=0):
"""
执行SQL命令
@param sql_command: SQL命令
@param param: SQL参数
@param pos: 索引
@return:
"""
logcm.print_info("Fetch many with Sql Command %s " % sql_command)
logcm.print_obj(param, "param")
logcm.print_obj(pos, "pos")
def fetchone(self, sql_command, param={}):
"""
执行SQL命令
@param sql_command: SQL命令
@param param: SQL参数
@return:
"""
self.cursor.execute(sql_command, param)
# 获取一条记录
one_data = self.cursor.fetchone()
logcm.print_obj(one_data, "one_data")
return one_data
def fetchall(self, sql_command, param={}):
"""
执行SQL命令
@param sql_command: SQL命令
@param param: SQL参数
@return:
"""
self.cursor.execute(sql_command, param)
# 获取多条记录
all_data = self.cursor.fetchall()
logcm.print_obj(all_data, "all_data")
return all_data
def get_latest_buildno(self, job_name):
"""
取得最近build版本号
@param job_name:JOB名称
@return: build版本号
"""
job = self.get_job(job_name)
lgb = job.get_last_good_build()
logcm.print_obj(lgb, "lgb")
return lgb.buildno
def Token(self):
url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken'
# corpid,corpsecret 为微信端获取
params = {
'corpid': self.corpid,
'corpsecret': self.corpsecret
}
url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken'
r = requests.get(url=url, params=params)
token = json.loads(r.text)['access_token']
logcm.print_obj(token, "token")
return token
def fetchmany(self, sql_command, param={}, pos=0):
"""
执行SQL命令
@param sql_command: SQL命令
@param param: SQL参数
@param pos: 索引
@return:
"""
self.cursor.execute(sql_command, param)
# 获取多条记录
many_data = self.cursor.fetmany(pos)
logcm.print_obj(many_data, "many_data")
return many_data
def set_val(self, key, val):
"""
设定Redis中指定Key的值
@param key: 指定Key
@param val: 指定值
@return: 是否成功
"""
# 取得Key对应值
logcm.print_info('Redis Set: %s --> %s' % (key, val))
result = self.rds.set(key, val)
logcm.print_obj(result, "result", show_header=False)
# 返回值
return result
def watch_job(self, job):
"""
监视JOB
@param job:JOB名称
@return: 无
"""
if job:
buildNo = job.get_last_buildnumber() + 1
consoleUrl = 'http://%s/jenkins/job/%s/%d/console' % (
self.cfg['host'], job.name, buildNo)
logcm.print_obj(consoleUrl, "Console")
print("<a class='linkBtn' href='%s' target='_blank'>查看控制台输出</a>" %
consoleUrl)
def get_server_time(self):
"""
取得服务器时间
:return: 服务器时间
"""
cmd = 'date "+%Y/%m/%d %H:%M:%S" '
lines = self.exe_cmd(cmd)
for line in lines:
line = line.strip("\r\n")
logcm.print_obj(line, "line")
server_time = datetime.datetime.strptime(line, '%Y/%m/%d %H:%M:%S')
logcm.print_obj(server_time, "date_time", show_header=False)
return server_time
return None
def get_last_build_param(self, job):
"""
重复最近一次构建参数
@param job:JOB
@return: 无
"""
lgb = job.get_last_good_build()
actions = lgb.get_actions()
params = {
pair['name']: pair['value']
for pair in actions["parameters"]
}
logcm.print_obj(params, "params", show_header=False)
return params
def md5_encrypt(input_bytes):
"""
对指定的字节列表进行MD5加密
@param input_bytes: 输入字节列表
@return: 加密后的字符串
"""
# 创建md5对象
hl = hashlib.md5()
hl.update(input_bytes)
logcm.print_info("MD5 encrypt bytes : %s" % input_bytes)
# 执行加密
result = hl.hexdigest()
logcm.print_obj(result, "md5-encrypt-result", show_header=False)
return result
def load_oracle_table(self, table_name):
"""
按照配置读取指定数据表名,载入模块数据
:param table_name: 数据表名
:return:数据表字典对象
"""
logcm.print_info("Loading table : %s ..." % table_name)
# 加载DB数据表模块信息
# 建立和数据库系统的连接
dbClient = DbOracleClient("oracle", self.cfg_db)
result = dbClient.getColumns(table_name)
logcm.print_obj(result, "result", show_table=True)
return result
def read_url(page_url, encoding):
"""
通过URL得到网页内容
@param page_url: 请求的网页地址
@param encoding: 网页编码
@return: 网页文本
"""
headers = {'User-Agent': random_agent()}
req = urllib.request.Request(url=page_url, headers=headers)
logcm.print_obj(req, 'req')
response = urllib.request.urlopen(req)
logcm.print_obj(response, 'response')
html = response.read().decode(encoding, 'ignore')
return html
def check_equal(obj1, obj2, title=""):
"""
判断两个对象的值是否相等
@param obj1: 对象1
@param obj2: 对象2
@param title: 对象描述
@return: 是否OK
"""
ok = (obj1 == obj2)
if ok:
logcm.print_info("Assert Ok for %s" % title)
else:
logcm.print_info("Assert Equal Fail for %s" % title, color='bold', fg='red')
logcm.print_obj(obj1, title + "-1", show_header=False)
logcm.print_obj(obj2, title + "-2", show_header=False)
return ok
def invoke_tag(self, job_name, tag_name, **kwargs):
"""
启动JOB
@param job_name:JOB名称ß
@param tag_name:GitTag名
@return: 无
"""
job = self.get_job(job_name)
# start build the job
params = {'GitVersion': tag_name}
logcm.print_obj(params, "params", show_header=False)
job.invoke(build_params=params)
logcm.print_info("Jenkins Job %s is invoked." % job_name)
self.watch_job(job)
def rsa_decrypt(prv_key_str, input_bytes):
"""
按照指定的私钥字符串进行RSA解密
@param prv_key_str: 私钥字符串
@param input_bytes: 输入字节列表
@return: 解密后的字节列表
"""
# 引入私钥字符串
prv_obj = rsa.importKey(prv_key_str)
prv_obj = PKCS1_v1_5.new(prv_obj)
# 执行解密
logcm.print_info("RSA decrypt bytes : %s" % input_bytes)
result = prv_obj.decrypt(input_bytes, '')
logcm.print_obj(result, "rsa-decrypt-result", show_header=False)
return result
def rsa_encrypt(pub_key_str, input_bytes):
"""
按照指定的公钥字符串进行RSA加密
@param pub_key_str: 公钥字符串
@param input_bytes: 输入字节列表
@return: 加密后的字节列表
"""
# 引入公钥字符串
pub_obj = rsa.importKey(pub_key_str)
pub_obj = PKCS1_v1_5.new(pub_obj)
# 执行加密
logcm.print_info("RSA encrypt bytes : %s" % input_bytes)
result = pub_obj.encrypt(input_bytes)
logcm.print_obj(result, "rsa-encrypt-result", show_header=False)
return result
def invoke(self, job_name, svn_url, task_no, **kwargs):
"""
启动JOB
@param job_name:JOB名称ß
@param svn_url:SVN的URL路径
@param task_no:任务号
@return: 无
"""
job = self.get_job(job_name)
# start build the job
params = {'url': svn_url, 'TASK_NO': task_no, 'svnPath': svn_url}
logcm.print_obj(params, "params", show_header=False)
job.invoke(build_params=params)
logcm.print_info("Jenkins Job %s is invoked." % job_name)
self.watch_job(job)
def xls_to_module(self, xls_path, sheet_name):
"""
按照配置读取指定Exel,指定Sheet名,载入模块数据
:param xls_path: Excel文件路径
:param sheet_name: Sheet名
:return:模块数据字典对象
"""
logcm.print_info("Loading xls : %s ..." % xls_path)
# 加载Excel模块信息
cfg_mdl = self.cfg_xls['Module']
mdl_list = xlscm.load_excel_dict(xls_path, sheet_name, **cfg_mdl)
# 模块信息为空判断
if mdl_list is None or len(mdl_list) == 0:
logcm.print_info("Module Info is not set!", fg='red')
sys.exit()
if len(mdl_list) != 1:
logcm.print_info("Module Info need only one!", fg='red')
sys.exit()
mdl_info = mdl_list[0]["Module"]
mdl = Module(**mdl_info)
# 加载接口信息
cfg_svc = self.cfg_xls['Services']
svc_list = xlscm.load_excel_dict(xls_path, sheet_name, **cfg_svc)
for svc in svc_list:
mdl.services.append(Service(svc))
# 加载Bean信息
cfg_bean = self.cfg_xls['Beans']
bean_list = xlscm.load_excel_dict(xls_path, sheet_name, **cfg_bean)
for bean in bean_list:
mdl.beans.append(Bean(bean))
# 加载Code信息
cfg_code = self.cfg_xls['Codes']
code_list = xlscm.load_excel_dict(xls_path, sheet_name, **cfg_code)
for code in code_list:
mdl.codes.append(Code(code))
logcm.print_obj(mdl, "mdl")
return mdl
def get_header(self, token_in_header=True):
"""
取得WEB访问Header
@param token_in_header: Token在Header中
@return: Header对象
"""
# Header对象
header = {
"terminal": self.cfg["terminal"],
"appType": self.cfg["appType"],
"appVersionNo": self.cfg["appVersionNo"],
"accessTerminal": self.cfg["accessTerminal"],
"reqTime": datecm.get_now_num()
}
# 在Header中存放Token
if token_in_header and self.token is not None:
header["token"] = self.token
logcm.print_obj(header, "header")
return header
def invoke_task(self, job_name, task_no, **kwargs):
"""
启动JOB
@param job_name:JOB名称
@param task_no:任务号
@return: 无
"""
# 取得JOB对象
job = self.get_job(job_name)
# 最近build参数
params = self.get_last_build_param(job)
# 替换任务号参数
params["TASK_NO"] = task_no
logcm.print_obj(params, "params", show_header=False)
job.invoke(build_params=params)
logcm.print_info("Jenkins Job %s is invoked." % job_name)
self.watch_job(job)
def get(self, key, convert=None):
"""
取得Redis中指定Key的值
@param key: 指定Key
@param convert: 指定转换方法
@return: Key对应值
"""
# 取得Key对应值
logcm.print_info('Redis Get by key: %s' % key)
try:
result = self.rds.get(key)
except Exception as e:
logcm.print_info("Exception : %s" % e)
result = None
# 不存在时
if result is None:
logcm.print_obj(result, "result", show_header=False)
return None
# 转换成字符串
result = result.decode()
# 无转换时直接返回
if convert is None:
logcm.print_obj(result, "result", show_header=False)
return result
# 执行转换
result = convert(result)
logcm.print_obj(result, "result", show_header=False)
return result
def load_diabetes():
"""
Load the diabetes dataset
@return: diabetes dataset by (data, target)
"""
# 文件路径
file_path_data = './cache/sk/sk_diabete_data.csv'
file_path_target = './cache/sk/sk_diabete_target.csv'
# 如果存在数据文件，则直接读取
if os.path.exists(file_path_data):
# 读取文件
df_data = pd.read_csv(file_path_data)
df_target = pd.read_csv(file_path_target)
return df_data.as_matrix(), df_target.as_matrix()
else:
filecm.makedir(file_path_data, by_file=True)
# 取得数据
diabetes = datasets.load_diabetes()
logcm.print_obj(diabetes.DESCR, 'diabetes.DESCR')
logcm.print_obj(diabetes.data, '样本原始数据集')
logcm.print_obj(diabetes.target, '样本目标数据集')
# 保存到文件
DataFrame(diabetes.data).to_csv(file_path_data, index=False)
DataFrame(diabetes.target).to_csv(file_path_target, index=False)
return diabetes.data, diabetes.target
