def _do_benchmark_copy_logs(src_dir, workload_name, date_str): with settings(warn_only=True): get( tar( src_dir, '%s-stats__%s__%s.log' % (workload_name, state.env['host'], date_str))) get( tar( src_dir, '%s__%s__%s.log' % (workload_name, state.env['host'], date_str)))
def _do_cassandra_copy_logs(workload_name, keyspace): cfg_dir = '/etc/cassandra/' logs_dir = '/var/log/cassandra' log = 'system.log' table = '%s.test_table' % keyspace workload = workload_name prefix = '%s__%s' % (workload, state.env['host']) with settings(warn_only=True): cfstats_out = '%s__cfstats.txt' % prefix run('nodetool cfstats %s > %s' % (table, cfstats_out)) get(cfstats_out) run('rm -f ' + cfstats_out) tpstats_out = '%s__tpstats.txt' % prefix run('nodetool tpstats > %s' % tpstats_out) get(tpstats_out) run('rm -f ' + tpstats_out) info_out = '%s__info.txt' % prefix run('nodetool info > %s' % info_out) get(info_out) run('rm -f ' + info_out) system_out = tar(logs_dir, log, '%s__%s.tar' % (prefix, log)) get(system_out) sudo('rm -f ' + system_out) cassandra_yaml = 'cassandra.yaml' cassandra_env = 'cassandra-env.sh' get(cfg_dir + cassandra_yaml, '%(host)s/' + prefix + '__' + cassandra_yaml) get(cfg_dir + cassandra_env, '%(host)s/' + prefix + '__' + cassandra_env)
def packet_lr(self): #get model print_log("client packeting model data.\n") if self.model_path == '' or self.feature_path == '': return False,'model path or feature path null' rc = self.get_model_from_hdfs(self.model_path,self.feature_path,self.model_name) if rc is False: return False,'get model from hdfs error' #modify config rc = self.modify_config(self.model_name) if rc is False: return False,'modify config error' #get release so lib self.get_release_so() online_path=os.path.join(self.online_path,self.service_name,self.version); tar_path=os.path.join(online_path,self.version) shutil.move(self.service_conf_path,os.path.join(tar_path,"./config")) shutil.move(self.data_path,os.path.join(tar_path,"./data")) shutil.move(self.release_lib_path,os.path.join(tar_path,"./lib")) tar_file = util.tar(tar_path,self.logger) util.del_dir(tar_path,self.logger) #res = util.get_file_md5(tar_file) res=util.gen_file_md5(online_path,tar_file,self.version,self.logger) #upload to monitor cmd = "rsync -r %s %s" %(os.path.join(self.online_path,self.service_name),self.monitor) print_log("packet upload cmd: %s\n" % cmd) rc,stdout,stderr= util.run_shell(cmd,self.logger) if rc is False: return False,'upload model error' self.logger.info("success upload model %s to monitor %s,model version %s",self.model_name,self.monitor,self.version) print_log("client finish packet: %s\n" % tar_file) return True, self.version
def packet(self): #get model print("client begin packet.") rc = self.get_model_from_hdfs(self.model_path,self.feature_path,self.model_name) if(rc is False): return False #modify config rc = self.modify_config(self.model_name) if(rc is False): return False #get release so lib self.get_release_so() online_path=os.path.join(self.online_path,self.service_name,self.version); tar_path=os.path.join(online_path,self.version) shutil.move(self.service_conf_path,os.path.join(tar_path,"./config")) shutil.move(self.data_path,os.path.join(tar_path,"./data")) shutil.move(self.release_lib_path,os.path.join(tar_path,"./lib")) tar_file = util.tar(tar_path,self.logger) util.del_dir(tar_path,self.logger) #res = util.get_file_md5(tar_file) res=util.gen_file_md5(online_path,tar_file,self.version,self.logger) #upload to monitor cmd = "rsync -r %s %s" %(os.path.join(self.online_path,self.service_name),self.monitor) print("packet upload cmd: %s" % cmd) rc,stdout,stderr= util.run_shell(cmd,self.logger) if(rc is False): return False self.logger.info("success upload model %s to monitor %s,model version %s",self.model_name,self.monitor,self.version) print("client finish packet.") return self.version
def packet_test(self): print("client begin packet.") online_path=os.path.join(self.online_path,self.service_name,self.version); tar_path=os.path.join(online_path,self.version) src_service_conf_path = os.path.join(self.service_conf_path,'service.conf') print("sevice conf path: %s" % src_service_conf_path) #all file,models.conf put in config data release,only service.conf modify demo_version rc = util.text_replace(src_service_conf_path,'demo_version',self.version,self.logger) if(rc is False): print("text replace service conf version failed") return False #shutil.move(self.service_conf_path,os.path.join(tar_path,"./config")) #shutil.move(self.data_path,os.path.join(tar_path,"./data")) #shutil.move(self.release_lib_path,os.path.join(tar_path,"./lib")) rc,stderr = util.copy_dir(self.service_conf_path,os.path.join(tar_path,"./config"),self.logger) if(rc is False): print(stderr) return False rc,stderr = util.copy_dir(self.data_path,os.path.join(tar_path,"./data"),self.logger) if(rc is False): print(stderr) return False rc,stderr = util.copy_dir(self.release_lib_path,os.path.join(tar_path,"./lib"),self.logger) if(rc is False): print(stderr) return False tar_file = util.tar(tar_path,self.logger) util.del_dir(tar_path,self.logger) res=util.gen_file_md5(online_path,tar_file,self.version,self.logger) #upload to monitor cmd = "rsync -r %s %s" %(os.path.join(self.online_path,self.service_name),self.monitor) print("packet upload cmd: %s" % cmd) #rc,stdout,stderr= util.run_shell(cmd,self.logger) #if(rc is False): # return False self.logger.info("success upload model %s to monitor %s,model version %s",self.model_name,self.monitor,self.version) print("client finish packet.") return self.version
def packet_dnn(self): dst_path=os.path.join(self.data_path, self.model_name, "version") f_path = os.path.join(self.data_path, self.model_name) util.mkdir(dst_path,self.logger) print_log("client begin packeting model\n") #get model rc = util.hdfs_file_copy(self.model_path,dst_path,False,"hadoop",self.logger) if rc is False: print_log("Fail to get %s from hdfs" % self.model_path) return False rc = util.hdfs_file_copy(self.feature_path,f_path,False,"hadoop",self.logger) if rc is False: print_log("Fail to get %s from hdfs" % self.feature_path) return False #modify config rc = self.modify_config(self.model_name) if rc is False: return False,'modify config error' #get release so lib self.get_release_so() online_path=os.path.join(self.online_path,self.service_name,self.version); tar_path=os.path.join(online_path,self.version) shutil.move(self.service_conf_path,os.path.join(tar_path,"./config")) shutil.move(self.data_path,os.path.join(tar_path,"./data")) shutil.move(self.release_lib_path,os.path.join(tar_path,"./lib")) tar_file = util.tar(tar_path,self.logger) util.del_dir(tar_path,self.logger) #res = util.get_file_md5(tar_file) res=util.gen_file_md5(online_path,tar_file,self.version,self.logger) #upload to monitor cmd = "rsync -r %s %s" %(os.path.join(self.online_path,self.service_name),self.monitor) print_log("packet upload cmd: %s\n" % cmd) rc,stdout,stderr= util.run_shell(cmd,self.logger) if rc is False: return False,'upload model error' self.logger.info("success upload model %s to monitor %s,model version %s",self.model_name,self.monitor,self.version) print_log("client finish packet: %s\n" % tar_file) return True, self.version
def mongodb_copy_logs(): files = ['config.log', 'rs0.log', 'rs1.log'] for file in files: log = tar('/disk1/mongodb-logs', file) get(log)