def _do_benchmark_copy_logs(src_dir, workload_name, date_str):
with settings(warn_only=True):
get(
tar(
src_dir, '%s-stats__%s__%s.log' %
(workload_name, state.env['host'], date_str)))
get(
tar(
src_dir, '%s__%s__%s.log' %
(workload_name, state.env['host'], date_str)))
def _do_cassandra_copy_logs(workload_name, keyspace):
cfg_dir = '/etc/cassandra/'
logs_dir = '/var/log/cassandra'
log = 'system.log'
table = '%s.test_table' % keyspace
workload = workload_name
prefix = '%s__%s' % (workload, state.env['host'])
with settings(warn_only=True):
cfstats_out = '%s__cfstats.txt' % prefix
run('nodetool cfstats %s > %s' % (table, cfstats_out))
get(cfstats_out)
run('rm -f ' + cfstats_out)
tpstats_out = '%s__tpstats.txt' % prefix
run('nodetool tpstats > %s' % tpstats_out)
get(tpstats_out)
run('rm -f ' + tpstats_out)
info_out = '%s__info.txt' % prefix
run('nodetool info > %s' % info_out)
get(info_out)
run('rm -f ' + info_out)
system_out = tar(logs_dir, log, '%s__%s.tar' % (prefix, log))
get(system_out)
sudo('rm -f ' + system_out)
cassandra_yaml = 'cassandra.yaml'
cassandra_env = 'cassandra-env.sh'
get(cfg_dir + cassandra_yaml,
'%(host)s/' + prefix + '__' + cassandra_yaml)
get(cfg_dir + cassandra_env,
'%(host)s/' + prefix + '__' + cassandra_env)
def packet_lr(self):
#get model
print_log("client packeting model data.\n")
if self.model_path == '' or self.feature_path == '':
return False,'model path or feature path null'
rc = self.get_model_from_hdfs(self.model_path,self.feature_path,self.model_name)
if rc is False:
return False,'get model from hdfs error'
#modify config
rc = self.modify_config(self.model_name)
if rc is False:
return False,'modify config error'
#get release so lib
self.get_release_so()
online_path=os.path.join(self.online_path,self.service_name,self.version);
tar_path=os.path.join(online_path,self.version)
shutil.move(self.service_conf_path,os.path.join(tar_path,"./config"))
shutil.move(self.data_path,os.path.join(tar_path,"./data"))
shutil.move(self.release_lib_path,os.path.join(tar_path,"./lib"))
tar_file = util.tar(tar_path,self.logger)
util.del_dir(tar_path,self.logger)
#res = util.get_file_md5(tar_file)
res=util.gen_file_md5(online_path,tar_file,self.version,self.logger)
#upload to monitor
cmd = "rsync -r %s %s" %(os.path.join(self.online_path,self.service_name),self.monitor)
print_log("packet upload cmd: %s\n" % cmd)
rc,stdout,stderr= util.run_shell(cmd,self.logger)
if rc is False:
return False,'upload model error'
self.logger.info("success upload model %s to monitor %s,model version %s",self.model_name,self.monitor,self.version)
print_log("client finish packet: %s\n" % tar_file)
return True, self.version
def packet(self):
#get model
print("client begin packet.")
rc = self.get_model_from_hdfs(self.model_path,self.feature_path,self.model_name)
if(rc is False):
return False
#modify config
rc = self.modify_config(self.model_name)
if(rc is False):
return False
#get release so lib
self.get_release_so()
online_path=os.path.join(self.online_path,self.service_name,self.version);
tar_path=os.path.join(online_path,self.version)
shutil.move(self.service_conf_path,os.path.join(tar_path,"./config"))
shutil.move(self.data_path,os.path.join(tar_path,"./data"))
shutil.move(self.release_lib_path,os.path.join(tar_path,"./lib"))
tar_file = util.tar(tar_path,self.logger)
util.del_dir(tar_path,self.logger)
#res = util.get_file_md5(tar_file)
res=util.gen_file_md5(online_path,tar_file,self.version,self.logger)
#upload to monitor
cmd = "rsync -r %s %s" %(os.path.join(self.online_path,self.service_name),self.monitor)
print("packet upload cmd: %s" % cmd)
rc,stdout,stderr= util.run_shell(cmd,self.logger)
if(rc is False):
return False
self.logger.info("success upload model %s to monitor %s,model version %s",self.model_name,self.monitor,self.version)
print("client finish packet.")
return self.version
def packet_test(self):
print("client begin packet.")
online_path=os.path.join(self.online_path,self.service_name,self.version);
tar_path=os.path.join(online_path,self.version)
src_service_conf_path = os.path.join(self.service_conf_path,'service.conf')
print("sevice conf path: %s" % src_service_conf_path)
#all file,models.conf put in config data release,only service.conf modify demo_version
rc = util.text_replace(src_service_conf_path,'demo_version',self.version,self.logger)
if(rc is False):
print("text replace service conf version failed")
return False
#shutil.move(self.service_conf_path,os.path.join(tar_path,"./config"))
#shutil.move(self.data_path,os.path.join(tar_path,"./data"))
#shutil.move(self.release_lib_path,os.path.join(tar_path,"./lib"))
rc,stderr = util.copy_dir(self.service_conf_path,os.path.join(tar_path,"./config"),self.logger)
if(rc is False):
print(stderr)
return False
rc,stderr = util.copy_dir(self.data_path,os.path.join(tar_path,"./data"),self.logger)
if(rc is False):
print(stderr)
return False
rc,stderr = util.copy_dir(self.release_lib_path,os.path.join(tar_path,"./lib"),self.logger)
if(rc is False):
print(stderr)
return False
tar_file = util.tar(tar_path,self.logger)
util.del_dir(tar_path,self.logger)
res=util.gen_file_md5(online_path,tar_file,self.version,self.logger)
#upload to monitor
cmd = "rsync -r %s %s" %(os.path.join(self.online_path,self.service_name),self.monitor)
print("packet upload cmd: %s" % cmd)
#rc,stdout,stderr= util.run_shell(cmd,self.logger)
#if(rc is False):
# return False
self.logger.info("success upload model %s to monitor %s,model version %s",self.model_name,self.monitor,self.version)
print("client finish packet.")
return self.version
def packet_dnn(self):
dst_path=os.path.join(self.data_path, self.model_name, "version")
f_path = os.path.join(self.data_path, self.model_name)
util.mkdir(dst_path,self.logger)
print_log("client begin packeting model\n")
#get model
rc = util.hdfs_file_copy(self.model_path,dst_path,False,"hadoop",self.logger)
if rc is False:
print_log("Fail to get %s from hdfs" % self.model_path)
return False
rc = util.hdfs_file_copy(self.feature_path,f_path,False,"hadoop",self.logger)
if rc is False:
print_log("Fail to get %s from hdfs" % self.feature_path)
return False
#modify config
rc = self.modify_config(self.model_name)
if rc is False:
return False,'modify config error'
#get release so lib
self.get_release_so()
online_path=os.path.join(self.online_path,self.service_name,self.version);
tar_path=os.path.join(online_path,self.version)
shutil.move(self.service_conf_path,os.path.join(tar_path,"./config"))
shutil.move(self.data_path,os.path.join(tar_path,"./data"))
shutil.move(self.release_lib_path,os.path.join(tar_path,"./lib"))
tar_file = util.tar(tar_path,self.logger)
util.del_dir(tar_path,self.logger)
#res = util.get_file_md5(tar_file)
res=util.gen_file_md5(online_path,tar_file,self.version,self.logger)
#upload to monitor
cmd = "rsync -r %s %s" %(os.path.join(self.online_path,self.service_name),self.monitor)
print_log("packet upload cmd: %s\n" % cmd)
rc,stdout,stderr= util.run_shell(cmd,self.logger)
if rc is False:
return False,'upload model error'
self.logger.info("success upload model %s to monitor %s,model version %s",self.model_name,self.monitor,self.version)
print_log("client finish packet: %s\n" % tar_file)
return True, self.version
def mongodb_copy_logs():
files = ['config.log', 'rs0.log', 'rs1.log']
for file in files:
log = tar('/disk1/mongodb-logs', file)
get(log)