def create_database():
conf = pyetc.load(r'sys.conf')
mydb = db.DB(host=config().host,
user=config().user,
passwd=config().passwd,
db=config().db,
port=int(config().port),
charset=config().charset)
return mydb
def main():
try:
config = pyetc.load(r'sys.conf')
for info in config.info_set:
send_info(info["brokerID"], info["userID"], info["report_name"],
config.max_send_num, info["table_record_path"],
config.trade_rec_report_url)
except Exception, msg:
print_log(u"error:%s" % (msg))
def __init__(self, config_file):
conf = pyetc.load(config_file, config_schema.ENV)
self.easy_install = config_schema.ConfigData({
'local' : '~/.octopus/easy_install/'
})
self.third_source = config_schema.ConfigData({
'prefix' : '~/third',
'build_dir' : '~/third/source'
})
etc = conf.CONFIG
if etc.has_key('easy_install') :
self.easy_install = etc.easy_install
if etc.has_key('third_source'):
self.third_source = etc.third_source
#self.artifactory_etc = etc.artifactory
self.artifactory = Artifactory(etc.artifactory)
def start(self):
self.t1 = time.time()
# check "configuration" file exist?
log_config = os.path.join(self.basePath, self.configurationFile)
if os.path.exists(log_config):
self.configuration = load(
os.path.join(self.basePath, self.configurationFile))
self.data = dict(self.config_all_default,
**self.configuration.config)
self.reloadConfig = threadReloadConfig(self)
self.reloadConfig.setDaemon(True)
self.reloadConfig.start()
else:
self.debugLog(
"Configuration File Not Found[%s]!! Use default Configuration..."
% log_config)
self.data = self.config_all_default
def main():
config = pyetc.load(r'system.ini')
test_parser_model = TestParserBase()
map_test_file = test_parser_model.find_test_file(config.test_result_path)
for contract in map_test_file:
test_result_files = map_test_file[contract]
if len(test_result_files) == 0:
continue
print contract
content = test_parser_model.create_result_web(template, contract, test_result_files)
full_result_path = test_parser_model.get_web_path(config.web_path, config.policy, contract)
f = open(full_result_path, "w+")
if f:
f.write(content)
f.close()
def load(self, program_path):
# 加载依赖信息
self.path = program_path
project_file = self.path + '/octopus.prj'
prj_conf = pyetc.load(project_file, project_schema.ENV)
self.depends = prj_conf.CONFIG.depends
self.package = prj_conf.CONFIG.package
self.packages = prj_conf.CONFIG.packages
# 从部署信息中加载版本信息.
deploy_metainfo = self.path + '/METAINFO/deploy.inf'
module2ver = read_deploy(deploy_metainfo)
for (mod_name, mod_ver) in module2ver.items():
if mod_name in self.depends:
self.depends[mod_name]['build'] = mod_ver
self.scripts = prj_conf.CONFIG.scripts
build_metainfo_file = self.path + '/METAINFO/build.inf'
self.buildmeta = Metainfo()
if os.path.exists(build_metainfo_file):
self.buildmeta.load(build_metainfo_file)
def main():
config = pyetc.load(r'system.ini')
#创建公共画板，每画一幅便要重新创建一块画板会降低速度
fig = plt.figure(figsize=(int(config.pic_width)/80, int(config.pic_height)/80))
# 首先通过根目录statistic文件找出目标文件前缀，再在各个目录中寻找标的文件解析（前提：先遍历根目录）
main_pre = ""
for root,dirs,files in os.walk(config.path):
if len(dirs) > 0:
for file in files:
if file.find("statistic", 0) != -1 and file.find(".log", 0) != -1:
pre_suffix = find_tar_pre_suffix(file)
if cmp(main_pre, "") == 0:
main_pre = pre_suffix[0]
tar_margin_file = "%s\\%smargin%s.log" % (root, main_pre, pre_suffix[1])
tar_profit_file = "%s\\%sclose_equity%s.log" % (root, main_pre, pre_suffix[1])
if os.path.exists(tar_margin_file):
ProduceMarginPic(fig, root, "", tar_margin_file, float(config.bar_alpha), config.bar_color, int(config.scale_num_x), int(config.scale_num_y), pre_suffix[1])
if os.path.exists(tar_profit_file):
ProduceProfitPic(fig, root, "", tar_profit_file, float(config.bar_alpha), config.bar_color, int(config.scale_num_x), int(config.scale_num_y), pre_suffix[1])
else:
index = root.rfind("\\", 0)
symbol = root[index+1:]
for file in files:
if file.find("statistic", 0) != -1 and file.find(".log", 0) != -1:
pre_suffix = find_tar_pre_suffix(file)
tar_margin_file = "%s\\%smargin%s.log" % (root, main_pre, pre_suffix[1])
tar_profit_file = "%s\\%sclose_equity%s.log" % (root, main_pre, pre_suffix[1])
if os.path.exists(tar_margin_file):
ProduceMarginPic(fig, root, symbol, tar_margin_file, float(config.bar_alpha), config.bar_color, int(config.scale_num_x), int(config.scale_num_y), pre_suffix[1])
if os.path.exists(tar_profit_file):
ProduceProfitPic(fig, root, symbol, tar_profit_file, float(config.bar_alpha), config.bar_color, int(config.scale_num_x), int(config.scale_num_y), pre_suffix[1])
#关闭画板
plt.close()
#!/usr/bin/python # -*- coding: utf-8 -*- import pyetc import os configfile = 'server.conf' conf = pyetc.load(configfile) host = conf.host localhost = conf.localhost port = conf.port bufsize = conf.bufsize support_client_type = conf.support_client_type
for d in dir_list:
m = re.search(r'(\d{4})-(\d{1,2})', d)
if not m:
continue
file_path = os.path.join(rootdir,d)
if not os.path.isdir(file_path):
continue
shutil.rmtree(file_path)
def my_proc(path, process_id):
process_scheduler(path, process_id)
if __name__ == '__main__':
multiprocessing.freeze_support()
import pyetc
conf = pyetc.load(r'sys.conf')
pool = multiprocessing.Pool(processes = conf.process_number)
process_id = 0
for p in conf.list_path:
pool.apply_async(my_proc, (p, process_id, )) #维持执行的进程总数为processes，当一个进程执行完毕后会添加新的进程进去
process_id = process_id + 1
pool.close()
pool.join() #调用join之前，先调用close函数，否则会出错。执行完close后不会有新的进程加入到pool,join函数等待所有子进程结束
# process_scheduler(p)
print("all done.")
# coding:utf-8
__author__ = 'xinchun.li'
import pyetc
conf = pyetc.load(r'D:\new_era\config\tech.conf')
def get(name, default=None):
value = getattr(conf, name, default)
if value is None:
raise ValueError('No config named %s' % name)
return value
if __name__ == '__main__':
print get('host', '1.1.1.1')
print get('port', 0)
print get('server')
def deploy(self, program_path, deploy_env):
"""
@program_path: 程序在本地的地址.
@env: 要部署环境
pre-condition:当前工作目录处在要部署的模块下.
从octopus.prj文件中读取依赖哪些库.
从METAINFO/deploy.inf中读取依赖的版本，如果没有该文件，则都用最新版本(build号最大的).
从deploy.conf文件中读取部署到哪里
拷贝模块自身到目标位置
拷贝依赖模块到目标位置（这个不好弄，依赖模块的路径怎么设定定?)
在远程机器上运行setup 脚本
在本机运行冒烟测试脚本(smoketest)
"""
print 'start deploy ' + program_path + ', env=' + deploy_env
REMOTE_PROGRAM_BASE = '~/program'
REMOTE_RUN_BASE = '~/service'
program_info = ProgramInfo(program_path)
depends = program_info.get_depends()
build_no = program_info.get_buildmeta().build_number
scripts = program_info.get_scripts()
# 读取部署位置文件,将文件拷贝过去.
deploy_conf_file = program_path + '/config/%s/deploy.conf' % (deploy_env)
deploy_conf = pyetc.load(deploy_conf_file, deploy_schema.ENV)
instances = deploy_conf.CONFIG['instance']
for instance in instances:
instance_name = instance['user'] + '@' + instance['host'] + ':'
instance_name = instance_name + instance['name']
fabric_env.host_string = instance['host']
print '=================start deploy %s =========' % (instance_name)
fabric_env.user = instance['user']
# 生成模版文件，打包成tar.gz.
homedir = CITool.get_homedir(pyshell.remote_shell)
run_path = REMOTE_RUN_BASE + '/' + instance['name']
run_path = CITool.normalize_path(run_path, homedir)
print run_path
remote_program_path = REMOTE_PROGRAM_BASE + '/' + instance['name']
if build_no > 0:
remote_program_path = '%s-%d' % (remote_program_path, build_no)
else:
# 如果取不到build号，则取当前时间.
cur_time = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))
remote_program_path = '%s-%s' % (remote_program_path, cur_time)
shell_ret = pyshell.remote_shell('if [ -d %s ] ;then exit 1;fi' % (
remote_program_path), warn_only=True)
if shell_ret.return_code == 0:
# 目录不存在，则重新打包拷贝.
pkg_file_full_path = self.make_package(instance, program_info,
program_path, deploy_env, run_path, remote_program_path)
run('mkdir -p %s' % (remote_program_path))
put(pkg_file_full_path, remote_program_path)
local('rm -f ' + pkg_file_full_path)
run('cd %s && tar -zxf %s' % (
remote_program_path, os.path.basename(pkg_file_full_path)))
# 安装依赖的模块.
print '===================<<%s>> start install depends========' % (
instance_name)
self.install_depends(depends, 'runtime', {'host':instance['host'],
'user':instance['user']})
# 按顺序执行脚本 install, stop, switch, start,check
# install -
print '===================<<%s>> start scripts:install========' % (
instance_name)
try:
self.run_script(instance, 'install', scripts,
remote_program_path, pyshell.remote_shell)
except Exception, e:
print 'step=install host=%s program=%s e=%s' % (
instance['host'], remote_program_path, e)
raise e
print '===================<<%s>> stop ========' % ( instance_name )
self.stop(run_path)
# switch - unlink link, ln -s install_path link
print '===================<<%s>> switch link========' % (
instance_name)
run('[ -d %s ] || mkdir -p %s' % (run_path, run_path))
run(('cd %s ; if [ -h program_old ]; then mv program_old program_old2;fi;'
+' if [ -h program ] ;then mv program program_old; fi ;'
+' ln -s %s program') % (
run_path, remote_program_path))
# 启动新的
print '===================<<%s>> start ========' % (
instance_name)
try:
self.start(run_path)
# 进行线上检查
print '===================<<%s>> check ========' % (
instance_name)
self.run_script(instance, 'check', scripts, remote_program_path,
run_path)
# 删除program_old2
print '===================<<%s>> delete old version ========' % (
instance_name)
ret = pyshell.remote_shell(('cd %s;'
+' if [ -h program_old2 ] ;then '
+'old2=`readlink program_old2`;'
+'old=`readlink program_old`;'
+'pp=`readlink program`;'
+ ' if [ "$old2" = "$old" -o "$old2" = "$pp" ];then rm -f program_old2;'
+'else rm -rf $old2 program_old2; fi;fi') % (run_path))
except Exception,e:
# 如果失败，回滚.回滚怎么重启老版本的服务？octopus.prj文件中的脚本可能发生了变化.
# 解决方案1:
# 1) 脚本不在octopus中配置，必须写死,但是解决不了多role问题
# 2) 或者重新读远程目录的octopus.prj文件, 仍然解决不了多role问题。
# 3) 必须生成一个部署meta文件,放在METAINFO/deploy.inf中，将role信息写入
# 另外依赖库怎么办？需要重新安装一遍吗？
# 看来最好的办法还是将所有的依赖库都放在自己的目录下.
# 1.一些第三方库整个维持一个版本(即depend_third每个程序只能有一个版本)
# 2. 模块下有个lib目录，放依赖的库和程序.
# 这样回滚的时候只需要修改链接就行.
print e
print '===================<<%s>> rollback ========' % (
instance_name)
# stop 正在部署的服务
self.stop(run_path)
run(('cd %s ; if [ -h program_fail ];then old_fail=`readlink program_fail`;pp=`readlink program`;'
+' if [ "$old_fail" = "$pp" ];then unlink program_fail; else rm -rf $old_fail program_fail;fi;'
+'fi; mv program program_fail; '
+'if [ -h program_old ] ; then mv program_old program; fi;'
+'if [ -h program_old2 ];then mv program_old2 program_old; fi') % (run_path))
# 切换目录,启动老版本的服务
try:
self.start(run_path)
except Exception, e:
print 'rollback faied ,reson=%s' % (e)
def get_config_sys():
config = py_etc.load(r'define\sys_conf_define.py')
return config
def get_config_DB():
config = py_etc.load(sys_conf_define.BASE_PATH + r'\define\db_define.py')
return config.DB
def config():
global g_config
if g_config is None:
g_config = pyetc.load(r'sys.conf')
return g_config