def configure_default_port(options):
from log import logger
from installer_io import InstallerIO
from configParser import ConfigParser
from text import text
io = InstallerIO(options.silent)
config_file = os.path.join(options.chorus_path, "shared/chorus.properties")
chorus_config = ConfigParser(config_file)
alpine_config_file = os.path.join(
options.chorus_path,
"shared/ALPINE_DATA_REPOSITORY/configuration/deploy.properties")
alpine_config = ConfigParser(alpine_config_file)
ports = ["server_port", "solr_port"]
menu = "\n".join(
str(i + 1) + ". %s: [default: %s]" %
(ports[i], chorus_config[ports[i]]) for i in xrange(0, len(ports)))
menu += "\n"
alpine_ports = ["alpine_port"]
menu += "\n".join(str(len(ports)+i+1) + ". %s: [default: %s]" % (alpine_ports[i], alpine_config[alpine_ports[i].replace("_", ".")]) \
for i in xrange(0, len(alpine_ports)))
menu += "\n%d. exit" % (len(ports) + len(alpine_ports) + 1)
num = io.require_menu(text.get("interview_question", "port_menu") % menu,
range(1,
len(ports) + len(alpine_ports) + 2),
default=len(ports) + len(alpine_ports) + 1)
if num in range(1, len(ports) + 1):
new_port = io.prompt_int(
text.get("interview_question", "change_port") % ports[num - 1],
default=int(chorus_config[ports[num - 1]]))
chorus_config[ports[num - 1]] = new_port
chorus_config.write(config_file)
logger.info("%s has successfully changed to %d" %
(ports[num - 1], new_port))
elif num in range(len(ports) + 1, len(ports) + 1 + len(alpine_ports)):
new_port = io.prompt_int(
text.get("interview_question", "change_port") %
alpine_ports[num - len(ports) - 1],
default=int(alpine_config[alpine_ports[num - len(ports) -
1].replace("_", ".")]))
alpine_config[alpine_ports[num - len(ports) - 1].replace(
"_", ".")] = new_port
alpine_config.write(alpine_config_file)
chorus_config["workflow.url"] = "http://%s:%d" % (
alpine_config["alpine.host"], new_port)
chorus_config.write(config_file)
logger.info("%s has successfully changed to %d" %
(alpine_ports[num - len(ports) - 1], new_port))
def __init__(self,basepath,confile="configs/config.xml"):
'''
Constructor
'''
self.basepath=basepath
self.cp = ConfigParser(confile)
self.fp=FolderParser(basepath)
self.fileitems=self.cp.getCheckItems()
if self.basepath==None or self.basepath=="":
self.basepath=self.fp.getLogPath()
#print "base path is " + self.basepath
self.internal_bt_p2=0
self.external_bt_p2=0
pass
def configure_default_port(options):
from log import logger
from installer_io import InstallerIO
from configParser import ConfigParser
from text import text
io = InstallerIO(options.silent)
config_file = os.path.join(options.chorus_path, "shared/chorus.properties")
chorus_config = ConfigParser(config_file)
alpine_config_file = os.path.join(options.chorus_path, "shared/ALPINE_DATA_REPOSITORY/configuration/deploy.properties")
alpine_config = ConfigParser(alpine_config_file)
ports = ["server_port", "solr_port"]
menu = "\n".join(str(i+1) + ". %s: [default: %s]" % (ports[i], chorus_config[ports[i]]) for i in xrange(0, len(ports)))
menu += "\n"
alpine_ports = ["alpine_port"]
menu += "\n".join(str(len(ports)+i+1) + ". %s: [default: %s]" % (alpine_ports[i], alpine_config[alpine_ports[i].replace("_", ".")]) \
for i in xrange(0, len(alpine_ports)))
menu += "\n%d. exit" % (len(ports) + len(alpine_ports) + 1)
num = io.require_menu(text.get("interview_question", "port_menu") % menu,
range(1, len(ports)+len(alpine_ports)+2), default=len(ports)+len(alpine_ports)+1)
if num in range(1, len(ports)+1):
new_port = io.prompt_int(text.get("interview_question", "change_port") % ports[num-1], default=int(chorus_config[ports[num-1]]))
chorus_config[ports[num-1]] = new_port
chorus_config.write(config_file)
logger.info("%s has successfully changed to %d" % (ports[num-1], new_port))
elif num in range(len(ports)+1, len(ports)+1+len(alpine_ports)):
new_port = io.prompt_int(text.get("interview_question", "change_port") % alpine_ports[num-len(ports)-1], default=int(alpine_config[alpine_ports[num-len(ports)-1].replace("_", ".")]))
alpine_config[alpine_ports[num-len(ports)-1].replace("_", ".")] = new_port
alpine_config.write(alpine_config_file)
chorus_config["workflow.url"] = "http://%s:%d" % (alpine_config["alpine.host"], new_port)
chorus_config.write(config_file)
logger.info("%s has successfully changed to %d" % (alpine_ports[num-len(ports)-1], new_port))
def __init__(self, app_path: Path, base_path: Path):
"""Initialises the configuration class
"""
_logger.debug(f'__init__ app_path "{app_path}"')
# Initialize
self._name = app_path.stem
self._base_path = base_path
# Folders
self._data_path = Path(self._base_path, 'data').resolve()
self._output_path = Path(self._base_path, 'output').resolve()
self._base_data_frame_path = Path(self._output_path, 'dataframes').resolve()
self._data_frame_path = Path(self._base_data_frame_path, date.today().strftime("%y-%m-%d")).resolve()
self._plot_base_path = Path(self._output_path, 'plots').resolve()
self._plot_path = Path(self._plot_base_path, date.today().strftime("%y-%m-%d")).resolve()
# Ensure directories pre-exist
self._data_path.mkdir(parents=True, exist_ok=True)
self._data_frame_path.mkdir(parents=True, exist_ok=True)
self._plot_path.mkdir(parents=True, exist_ok=True)
_logger.debug(f'data path: "{self._data_path}"'
f'data frame path: "{self._data_path}"'
f'plot path: "{self._plot_path}"')
self._config_plots = ConfigParser.parse(Path(_PLOTS_CONFIG_FILENAME))
def pipeline_endpoint():
data = request.json
parsed_data = getRepo('pipeline', data)
configUrl = parsed_data['config']
config = ConfigParser(configUrl).getConfig()
startCrow(parsed_data, config)
return 'OK'
def main():
data = request.json
parsed_data = getRepo(CROW_REPO, data)
configUrl = CROW_RAW_REPO + '/' + parsed_data['image'] + '/' + parsed_data['branch'] + '/crow.yaml'
config = ConfigParser(configUrl).getConfig()
startCrow(parsed_data, config)
return 'OK'
def enable_https(options):
from installer_io import InstallerIO
from chorus_executor import ChorusExecutor
from log import logger
from configParser import ConfigParser
from text import text
io = InstallerIO(options.silent)
config_file = os.path.join(options.chorus_path, "shared/chorus.properties")
chorus_config = ConfigParser(config_file)
if chorus_config.has_key("ssl.enabled") and chorus_config["ssl.enabled"].lower() == "true":
if not io.require_confirmation(text.get("interview_question", "https_question"), default="no"):
return
executor = ChorusExecutor(options.chorus_path)
server_key = os.path.join(os.path.join(options.chorus_path, "shared/server.key"))
server_csr = os.path.join(os.path.join(options.chorus_path, "shared/server.csr"))
executor.run("openssl genrsa -des3 -out %s 1024" % server_key)
ret = executor.call("openssl req -new -key %s -out %s" % (server_key, server_csr))
if ret != 0:
logger.error("failed to enable https, try again.")
return
server_key_org = os.path.join(os.path.join(options.chorus_path, "shared/server.key.org"))
executor.run("cp %s %s" % (server_key, server_key_org))
executor.run("openssl rsa -in %s -out %s" % (server_key_org, server_key))
server_crt = os.path.join(os.path.join(options.chorus_path, "shared/server.crt"))
executor.run("openssl x509 -req -days 365 -in %s -signkey %s -out %s" % \
(server_csr, server_key, server_crt))
port = io.prompt_int(text.get("interview_question", "https_port"), default=8443)
chorus_config["ssl.enabled"] = "true"
chorus_config["ssl_server_port"] = port
chorus_config["ssl_certificate"] = server_crt
chorus_config["ssl_certificate_key"] = server_key
chorus_config.write(config_file)
alpine_conf = os.path.join(options.chorus_path, "shared/ALPINE_DATA_REPOSITORY/configuration/alpine.conf")
with open(alpine_conf, "r") as f:
contents = f.read()
content = re.findall(r"chorus *{(.*?)}", contents, re.DOTALL)[0]
replace = "active = true\n" + "scheme = HTTPS\n" + "port = %d\n" % port
contents = contents.replace(content, replace)
with open(alpine_conf, "w") as f:
f.write(contents)
logger.info("https has been configured successfully on port %d" % port)
class DbConn(object):
__metaclass__ = Singleton
def __init__(self, conf=DEFAULTS['BACKEND_CFG']):
'''初始化日志模块、配置信息'''
self.config = ConfigParser()
self.config.read(conf)
self.dbconn_dict = {}
def oracle_conn_test(self, oracle_conn):
if oracle_conn:
try:
oracle_conn.ping()
except InterfaceError, err:
logging.warn(err.__class__.__name__ + ':' + str(err))
oracle_conn = None
return oracle_conn
class DbConn(object):
__metaclass__ = Singleton
def __init__(self, conf=DEFAULTS['BACKEND_CFG']):
'''初始化日志模块、配置信息'''
self.config = ConfigParser()
self.config.read(conf)
self.dbconn_dict = {}
def oracle_conn_test(self, oracle_conn):
if oracle_conn:
try:
oracle_conn.ping()
except InterfaceError, err:
logging.warn(err.__class__.__name__ + ':' + str(err))
oracle_conn = None
return oracle_conn
def main():
config = ConfigParser().config
molgenis_connector = MolgenisConnector(config['url'], config['account'],
config['password'])
qc = QualityChecker(molgenis_connector)
qc.check_collection_data()
qc.check_biobank_data()
qc.check_network_data()
qc.check_person_data()
qc.logs.close()
molgenis_connector.logout()
def __init__(self, logfolder):
'''
Constructor
'''
# store the real path to a file
self.storefile = "tmp/latest_log_path"
# static defined variable
self.dateinternal = "DATE_INTERNAL"
self.dateinternal_lastlog = "DATEINTERNAL_LASTLOG"
self.dateexternal = "DATE_EXTERNAL"
self.dateexternal_lastlog = "DATEEXTERNAL_LASTLOG"
# store the fade and real path map
self.mapfadereal = {}
self.initdata()
self.logfolder = logfolder
if self.logfolder == None or self.logfolder == "":
self.logfolder = self.getLogPath()
self.fullfilepaths = []
self.fullfolderpaths = []
self.cp = ConfigParser("configs/config.xml")
self.workpath()
def __init__(self, logfolder):
'''
Constructor
'''
# store the real path to a file
self.storefile = "tmp/latest_log_path"
# static defined variable
self.dateinternal = "DATE_INTERNAL";
self.dateinternal_lastlog = "DATEINTERNAL_LASTLOG";
self.dateexternal = "DATE_EXTERNAL";
self.dateexternal_lastlog = "DATEEXTERNAL_LASTLOG";
# store the fade and real path map
self.mapfadereal = {}
self.initdata()
self.logfolder = logfolder
if self.logfolder == None or self.logfolder == "":
self.logfolder = self.getLogPath()
self.fullfilepaths = []
self.fullfolderpaths = []
self.cp = ConfigParser("configs/config.xml")
self.workpath()
from xmlReader import XmlReader
from gitConfig import GitConfig
from jsonReader import JsonReader
from configParser import ConfigParser
# GitConfig.checkoutDev()
#release = XmlReader.readRelease('C:/MyProjects/terminaliniwifi.server/pom.xml')
#release_ = JsonReader.readRelease('C:/MyProjects/terminaliniwifi.client/package.json')
#print(release_)
#print(release)
# map = ConfigParser.getPathFormat(var)
# print(map)
listPlatforms = ConfigParser.getPlatform()
for platform in listPlatforms:
print(platform)
mapConfig = ConfigParser.getPathFormat(platform)
if 'path' in mapConfig:
endPath = mapConfig['path'] + '\pom.' + mapConfig['extension']
release = XmlReader.readRelease(endPath)
GitConfig.configureGit(mapConfig['path'])
GitConfig.checkoutDev()
GitConfig.executeRelease(release, endPath)
#GitConfig.upgradeRelease()
def __init__(self, conf=DEFAULTS['BACKEND_CFG']):
'''初始化日志模块、配置信息'''
self.config = ConfigParser()
self.config.read(conf)
self.dbconn_dict = {}
class LogCheck(object):
'''
classdocs
'''
def __init__(self,basepath,confile="configs/config.xml"):
'''
Constructor
'''
self.basepath=basepath
self.cp = ConfigParser(confile)
self.fp=FolderParser(basepath)
self.fileitems=self.cp.getCheckItems()
if self.basepath==None or self.basepath=="":
self.basepath=self.fp.getLogPath()
#print "base path is " + self.basepath
self.internal_bt_p2=0
self.external_bt_p2=0
pass
def checkFile(self,f):
if f == None:
return False
else:
if isLinux():
if f.find("/")==0:
pass
else:
f="/"+f
if self.basepath[len(self.basepath)-2:len(self.basepath)-1]=="/":
f=self.basepath[0:len(self.basepath)-2]+f
else:
f=self.basepath+f
else:
if f.find("\\")==0:
pass
else:
f="\\"+f
if self.basepath[len(self.basepath)-2:len(self.basepath)-1]=="\\":
f=self.basepath[0:len(self.basepath)-2]+f
else:
f=self.basepath+f
#print "check file path is " + f
if os.path.isfile(f):
return True
def getRealPath(self,f):
if isLinux():
if f.find("/")==0:
pass
else:
f="/"+f
if self.basepath[len(self.basepath)-2:len(self.basepath)-1]=="/":
f=self.basepath[0:len(self.basepath)-2]+f
else:
f=self.basepath+f
else:
if f.find("\\")==0:
pass
else:
f="\\"+f
if self.basepath[len(self.basepath)-2:len(self.basepath)-1]=="\\":
f=self.basepath[0:len(self.basepath)-2]+f
else:
f=self.basepath+f
return f
def checkFolder(self,f):
if f == None:
return False
else:
if isLinux():
if f.find("/")==0:
pass
else:
f="/"+f
if self.basepath[len(self.basepath)-2:len(self.basepath)-1]=="/":
f=self.basepath[0:len(self.basepath)-2]+f
else:
f=self.basepath+f
else:
if f.find("\\")==0:
pass
else:
f="\\"+f
if self.basepath[len(self.basepath)-2:len(self.basepath)-1]=="\\":
f=self.basepath[0:len(self.basepath)-2]+f
else:
f=self.basepath+f
if os.path.isdir(f):
return True
return False
pass
def fileCountIn(self,dir):
return sum([len(files) for root,dirs,files in os.walk(dir)])
# f.get("type"),f.get("ismust"),f.get("checkitem"),f.get("condition"),f.text
def check(self):
allPass=True
resultbase="check base path is :\n" + self.basepath + "\n\n"
for f in self.fileitems:
erroroccor=False
#realPath=self.fp.getRealPath(os.path.join(basepath,f.getPath()))
realPath=self.fp.getRealPath(f.getPath())
#result+=realPath
#print str(f.getPath()) + "real path is " + str(realPath)
# check if file exist
if f.getStype() == "file":
if self.checkFile(realPath):
pass
else:
erroroccor=True
result=realPath+" folder not exist! \n"
f.setCheckresult(result)
if int(f.getIsmust())==0:
result+="ommit"
pass
else:
allPass=False
#result+=" ismust== "+ str(f.getIsmust())+"\n"
continue
pass
elif f.getStype() == "folder":
if self.checkFolder(realPath):
pass
else:
erroroccor=True
result=realPath+" folder not exist!\n"
f.setCheckresult(result)
if int(f.getIsmust())==0:
result+="ommit"
pass
else:
allPass=False
#result+=" ismust== "+ str(f.getIsmust())+"\n"
continue
pass
else:
print "exception please check if config error accour\n"
continue
# record ismust
# checkitem
fi=f.getCheckItemNum()
#print "file count is " + str(fi)
#print "f is must=="+str(f.getIsmust())
#print "f is path=="+str(f.getPath()) + str(realPath)
count=self.fileCountIn(self.getRealPath(realPath))
#print f.getPath()+" check num is " +str(fi)+"count is " + str(count)+ str(erroroccor)
if fi==1:
# check subfiles
#print "fi==1" + realPath + "count " + str(count)
if count>0:
pass
else:
if int(f.getIsmust())==0:
result=realPath+" folder has no subfiles ommit\n"
f.setCheckresult(result)
pass
else:
erroroccor=True
result=realPath+" folder has no subfiles\n"
f.setCheckresult(result)
allPass=False
pass
pass
pass
# check condition > < = logical
elif fi==2:
#print realPath
if f.doCondition(count):
pass
else:
if int(f.getIsmust())==0:
result=realPath+" check condtion count "+f.getCondition()+" can not meet the desire,ommit\n"
f.setCheckresult(result)
pass
else:
erroroccor=True
result=realPath+" check condition count "+f.getCondition()+" can not meet the desire \n"
f.setCheckresult(result)
allPass=False
pass
pass
elif fi==3:
#if realPath[-2:]=="bt" or realPath[-2:]=="p2":
# print "deal with"
# foldersize check
size = getdirsize(self.getRealPath(realPath))
#print " " + str(self.getRealPath(realPath)) + " size is " + str(size)
if size>0:
pass
else:
if int(f.getIsmust())==0:
result=realPath+" size is 0 can not meet the desire,ommit\n"
f.setCheckresult(result)
pass
else:
erroroccor=True
result=realPath+" size is 0 can not meet the desire,ommit\n"
f.setCheckresult(result)
allPass=False
pass
else:
pass
if erroroccor:
#result=realPath+result
erroroccor=False
# dic store result <checktype results>
results_dic={}
# get result from items
for f in self.fileitems:
if f.getCheckresult() != "":
if results_dic.get(f.getChecktype(),"notexist") == "notexist":
results_dic[f.getChecktype()]=f.getCheckresult()
else:
results_dic[f.getChecktype()]=f.getCheckresult() + results_dic.get(f.getChecktype())
# get result from dict
result=resultbase
for (d,x) in results_dic.items():
result +=d + " check result:\n" +str(x) + "\n"
return allPass,result
pass
class FolderParser(object):
'''
classdocs
'''
def __init__(self, logfolder):
'''
Constructor
'''
# store the real path to a file
self.storefile = "tmp/latest_log_path"
# static defined variable
self.dateinternal = "DATE_INTERNAL";
self.dateinternal_lastlog = "DATEINTERNAL_LASTLOG";
self.dateexternal = "DATE_EXTERNAL";
self.dateexternal_lastlog = "DATEEXTERNAL_LASTLOG";
# store the fade and real path map
self.mapfadereal = {}
self.initdata()
self.logfolder = logfolder
if self.logfolder == None or self.logfolder == "":
self.logfolder = self.getLogPath()
self.fullfilepaths = []
self.fullfolderpaths = []
self.cp = ConfigParser("configs/config.xml")
self.workpath()
def initdata(self):
self.mapfadereal[self.dateinternal] = ""
self.mapfadereal[self.dateinternal_lastlog] = ""
self.mapfadereal[self.dateexternal] = ""
self.mapfadereal[self.dateexternal_lastlog] = ""
pass
def printData(self):
print self.mapfadereal[self.dateinternal]
print self.mapfadereal[self.dateinternal_lastlog]
print self.mapfadereal[self.dateexternal]
print self.mapfadereal[self.dateexternal_lastlog]
# return the log path if no basepath passed
def getLogPath(self):
if self.logfolder == None or self.logfolder == "":
f = open(self.storefile, "r")
path = f.readline()
# get current path
return path.strip()
else:
return self.logfolder
# return report folder
def getReportPath(self):
logpath = self.getLogPath()
if isLinux():
result = logpath.split("/")
else:
result = logpath.split("\\")
path = result[len(result) - 1]
if path == "":
path = result[len(result) - 2]
if path == "":
path = "xx_unknow_"
# get current path
os.path.abspath('.')
if isLinux():
return os.path.abspath('.') + "/report/" + path.split("_")[1]
else:
return os.path.abspath('.') + "\\report\\" + path.split("_")[1]
def workpath(self):
for parent, dirnames, filenames in os.walk(self.logfolder): # 三个参数：分别返回1.父目录 2.所有文件夹名字（不含路径） 3.所有文件名字
for dirname in dirnames: # 输出文件夹信息
# print "parent is:" + parent
# check if endwith external_storage
# check if endwith external_storage
# print "dirname is" + dirname
# internal_storage
# internal_storage/last_log
# internal_storage/2014-06-03-19-27-04
if parent.endswith('external_storage'):
if dirname != 'last_log' and dirname.find("-")>0:
self.mapfadereal[self.dateexternal] = dirname
elif parent.endswith('last_log'):
if parent.find("external_storage") > 0:
self.mapfadereal[self.dateexternal_lastlog] = dirname
elif parent.find("internal_storage") > 0:
self.mapfadereal[self.dateinternal_lastlog] = dirname
pass
if parent.endswith('internal_storage'):
if dirname != 'last_log' and dirname.find("-")>0:
self.mapfadereal[self.dateinternal] = dirname
self.fullfolderpaths.append(os.path.join(parent, dirname))
#self.printData()
for filename in filenames: # 输出文件信息
# print "parent is"+ parent
# print "filename is:" + filename
# print "the full name of the file is:" + os.path.join(parent,filename) #输出文件路径信息
self.fullfilepaths.append(os.path.join(parent, filename))
def getmap(self):
return self.mapfadereal
def getfiles(self):
return self.fullfilepaths
def getfilesToStr(self):
return ",".join(self.fullfilepaths)
def getfolder(self):
return self.fullfolderpaths
def getfolderToStr(self):
return ",".join(self.fullfolderpaths)
def printmap(self):
print self.mapfadereal
def printfiles(self):
print self.fullfilepaths
def printdirs(self):
print self.fullfolderpaths
def getFileCount(self):
return len(self.fullfilepaths)
def getFolderCount(self):
return len(self.fullfolderpaths)
pass
def getAnrFiles(self):
typename = "anr"
self.getFilesBy(typename)
pass
# return the string list of files with "," split
def getFilesBy(self, typeName):
files = self.cp.getProblemFiles(typeName)
tmprefiles = []
#print files
# get all path meeted files
for i in self.fullfilepaths:
for f in files:
#print "xx"+f.getPath()
#print "yy"+self.getRealPath(f.getPath())
pp=""
if not isLinux():
pp=f.getPath().replace("/","\\")
else:
pp=f.getPath().replace("\\","/")
#print self.getRealPath(pp)
#print i
if i.find(self.getRealPath(pp)) >= 0:
# check items
chk = f.getCheckitem()
#print "chk" + str(chk)
if chk == "" or chk == None:
tmprefiles.append(i)
else:
for ii in chk.split(","):
if i.find(ii) >= 0:
tmprefiles.append(i)
break
break
return ",".join(tmprefiles)
pass
def __checkComplete(self, f):
realPath = f
if realPath.find(self.dateinternal) >= 0:
return None
if realPath.find(self.dateinternal_lastlog) >= 0:
return None
if realPath.find(self.dateexternal) >= 0:
return None
if realPath.find(self.dateexternal_lastlog) >= 0:
return None
'''
if isLinux():
if f.find("/")==0:
pass
else:
f="/"+f
if self.logfolder[len(self.logfolder)-2:len(self.logfolder)-1]=="/":
f=self.logfolder[0:len(self.logfolder)-2]+f
else:
f=self.logfolder+f
else:
if f.find("\\")==0:
pass
else:
f="\\"+f
if self.logfolder[len(self.logfolder)-2:len(self.logfolder)-1]=="\\":
f=self.logfolder[0:len(self.logfolder)-2]+f
else:
f=self.logfolder+f
'''
return f
#return os.path.join(self.logfolder, realPath)
# return null if file path is not exist
def getRealPath(self, fadepath):
result = fadepath
if result == "" or result == None:
return ""
for (k, v) in self.mapfadereal.items():
# print k,v
if v == "":
result = result.replace(k, "nofolderfound")
else:
result = result.replace(k, v)
# print result
pass
return self.__checkComplete(result)
pass
# coding=utf-8
import logging
from configParser import ConfigParser
from mysql import connector
from Config.defaults import DEFAULTS
DB_CONFIG = ConfigParser()
DB_CONFIG.read(DEFAULTS['BACKEND_CFG'])
dbconfig = DB_CONFIG.items('KVM_MYSQL_TEST')
def get_mysql_conn(cid='KVM_MYSQL_TEST'):
#mysql_conn = self.dbconn_dict.setdefault(cid, None)
cfg = dict(DB_CONFIG.items(cid))
cfg['port'] = int(cfg['port'])
mysql_conn = connector.connect(**cfg)
return mysql_conn
# if not mysql_conn:
# try:
# cfg = dict(self.config.items(cid))
# cfg['port'] = int(cfg['port'])
# mysql_conn = connector.connect(**cfg)
# except Exception, err:
# logging.error(err)
# raise err # 连接异常时直接抛出错误
# else:
# self.dbconn_dict[cid] = mysql_conn
# logging.info('MYSQL - CONNECTION %s - SUCCEED' % mysql_conn)
def main():
global sPath
global alive
global running
global mConfig, mMap, mExplain
global display, display1, display2
global log
global nRoom
global aColour
global localFile
global aBlock
global notifyMode
global beatClock
# use crafted display function to ease the migration from python3 to python2 and to accommodate to different terminal coding in different system
# display1 is normal displayer, while display2 is a separate displayer running in special thread, implementing the display interval
# in each case, display1 shall be a instant displayer; thus, use display1 to output diagnostic message
display1 = Displayer(0).display
display = display1
mConfigBak = mConfig.copy()
try:
parser1 = ConfigParser(mConfig, mExplain, mMap,
'display danmu message in bilibili live')
useCLI = True if len(sys.argv) > 1 else False
if (not os.path.exists(sPath)):
sDir = os.path.split(sys.argv[0])[0]
sFile = os.path.join(sDir, sPath)
if (os.path.exists(sFile)):
sPath = sFile
else:
display1('配置文件 {} 不存在'.format(sPath))
sPath = None
# parse configuration from file and from command line option
mData = parser1.parse(sPath, useCLI)
mConfig = mData
except Exception as e:
display1('读取配置出错：', e, sep='\n')
display1('退回默认配置')
mConfig = mConfigBak
if (mConfig['nDelay'] > 0):
# danmu message display interval is enabled, using threaded displayer
display2 = Displayer(1, mConfig['nDelay']).display
display = display2
aColour = [(x + 30 if x < 10 else x + 80) for x in mConfig['aColour']]
if (mConfig['verbose']):
log = display1
else:
def log(*aArgs, **mArgs):
pass
if (mConfig['block']):
# it seems that two format of flooding messages are existing
aBlock = ['bilibili-(゜-゜)つロ乾杯~', '- ( ゜- ゜)つロ 乾杯~ - bilibili']
if (mConfig['notify']):
notifyMode = 2
else:
notifyMode = 1
log(mConfig)
nRoom = mConfig['nRoom'] or int(input('room ID:'))
running = True
socket.setdefaulttimeout(10)
while running:
try:
try:
sServer, nRoom, sHoster, sTitle = getRoom(nRoom)
except urllib.error.HTTPError as e:
if (e.code == 404):
display1('找不到该房间，请重新输入房间号')
nRoom = int(input('room ID:'))
continue
else:
raise
if (mConfig['write']):
sTime = time.strftime('%m%d_%H%M%S-')
sName = sHoster + '-' + sTitle
sName = re.sub(r'[^\w_\-.()]', '-', sName)
sFileName = '{}{}.txt'.format(sTime, sName)
localFile = open(sFileName, 'a', encoding='utf-8')
log('弹幕服务器 ' + sServer)
aAddr1 = (sServer, 788)
sock1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock1.connect(aAddr1)
except TimeoutError as e:
sock1.close()
display1('到弹幕服务器的连接失败，尝试更换地址')
if (sServer == 'livecmt-1.bilibili.com'):
sServer = 'livecmt-2.bilibili.com'
else:
sServer = 'livecmt-1.bilibili.com'
log('弹幕服务器 ' + sServer)
aAddr1 = (sServer, 788)
sock1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock1.connect(aAddr1)
log('地址为 ', *sock1.getpeername())
nUid = int(100000000000000 + 200000000000000 * random.random())
# a random meaningless user ID
#bPayload = b'{"roomid":%d,"uid":%d}' % (nRoom, nUid);
bPayload = ('{"roomid":%d,"uid":%d}' %
(nRoom, nUid)).encode('utf-8')
nLength = len(bPayload) + 16
bReq = struct.pack('>IIII', nLength, 0x100001, 0x7, 0x1)
bReq += bPayload
sock1.sendall(bReq)
alive = True
bHeartBeat = struct.pack('>IIII', 0x10, 0x100001, 0x2, 0x1)
sock1.sendall(bHeartBeat)
# send heartbeat message per 30 seconds
interval = SetInterval(lambda: (sock1.sendall(bHeartBeat)), 30)
interval.start()
# capture CR in stdin to send hearbeat in order to fetch freshed online count
if (not beatClock):
beatClock = interval.clock
t = threading.Thread(target=notify)
t.daemon = 1
t.start()
else:
beatClock = interval.clock
handler2(sock1)
except (socket.timeout, TimeoutError) as e:
display1('连接超时，重试...')
continue
except SocketDied as e:
display1('连接被关闭，程序重启...')
continue
except BaseException as e:
if (isinstance(e, KeyboardInterrupt)):
display1('程序退出')
running = False
elif (sys.version[0] == '3'
and isinstance(e, ConnectionResetError)):
# ConnectionResetError is not supported in python2
display1(e)
display1('到服务器的连接被断开，尝试重新连接...')
continue
else:
with open('danmu_error.log', 'ab') as f1:
# record error log
f1.write(('\n' + (str(e))).encode('utf-8'))
raise
finally:
alive = False
if ('interval' in locals()): interval.stop()
if ('sock1' in locals()): sock1.close()
if (localFile):
display1('弹幕已保存到文件 {}'.format(localFile.name))
localFile.close()
def __init__(self, conf=DEFAULTS['BACKEND_CFG']):
'''初始化日志模块、配置信息'''
self.config = ConfigParser()
self.config.read(conf)
self.dbconn_dict = {}
from getData import GetData
from configParser import ConfigParser
import os
import time
import signal
import sys
osVer = platform.system()
try:
os.mkdir(saneConf.temp)
except OSError:
pass
if osVer == "Darwin":
import externs
conf = ConfigParser("conf/client/osx.cfg").get()
elif osVer == "Windows":
import externs
conf = ConfigParser("conf/client/windows.cfg").get()
elif osVer == "Linux":
import externs
conf = ConfigParser("conf/client/linux.cfg").get()
def exit(signum, frame):
externs.setWallpaper(bg)
sys.exit()
signal.signal(signal.SIGINT, exit)
def test_config_rules(self):
config = ConfigParser("conf/client/test.cfg").get()
for x in range(len(config.rules)):
self.assertEqual(config.rules[x].name, self.goodConf.rules[x].name)
self.assertEqual(config.rules[x].rule, self.goodConf.rules[x].rule)
def test_config_header(self):
config = ConfigParser("conf/client/test.cfg").get()
self.assertEqual("Test Header", config.header)
# coding=utf-8
import logging
from configParser import ConfigParser
from mysql import connector
from Config.defaults import DEFAULTS
DB_CONFIG = ConfigParser()
DB_CONFIG.read(DEFAULTS['BACKEND_CFG'])
dbconfig = DB_CONFIG.items('KVM_MYSQL_TEST')
def get_mysql_conn(cid='KVM_MYSQL_TEST'):
#mysql_conn = self.dbconn_dict.setdefault(cid, None)
cfg = dict(DB_CONFIG.items(cid))
cfg['port'] = int(cfg['port'])
mysql_conn = connector.connect(**cfg)
return mysql_conn
# if not mysql_conn:
# try:
# cfg = dict(self.config.items(cid))
# cfg['port'] = int(cfg['port'])
# mysql_conn = connector.connect(**cfg)
# except Exception, err:
# logging.error(err)
# raise err # 连接异常时直接抛出错误
# else:
# self.dbconn_dict[cid] = mysql_conn
# logging.info('MYSQL - CONNECTION %s - SUCCEED' % mysql_conn)
gc.loadHeirarchy(application='atg', datacenter='me')
print(gc.paths)
print ("***************** CONSOLIDATED JSON ITERATIVE ***********************")
gc.buildConsolidatedJSON()
gc.printOrderedJSON()
print ("***************** CONSOLIDATED JSON RECURSIVE ***********************")
gc.buildConsolidatedJSONRecursive()
gc.printOrderedJSON()
gc.writeOrderedJSON()
print ("***************** CONFIG PARSER FILTER JSON ***********************")
cp=ConfigParser()
cp.loadJSONDict(gc.jsonMerged)
cp.loadGlobalConfig(dataPath+"/globalConfigFull.json")
cp.filterJSON('atg')
cp.printOrderedJSON()
cp.writeOrderedJSON(dataPath)
print ("***************** INTERPOLATE KEYS ***********************")
i=Interpolate()
i.loadGlobalConfig(dataPath+"/globalConfig.tmpl")
i.interpolateDict()
i.printOrderedJSON()
i.writeOrderedJSON(dataPath)
#i.interpolateTemplate(dataPath+"/sample.txt.tmpl")
i.interpolate(dataPath+"/multidir")
class FolderParser(object):
'''
classdocs
'''
def __init__(self, logfolder):
'''
Constructor
'''
# store the real path to a file
self.storefile = "tmp/latest_log_path"
# static defined variable
self.dateinternal = "DATE_INTERNAL"
self.dateinternal_lastlog = "DATEINTERNAL_LASTLOG"
self.dateexternal = "DATE_EXTERNAL"
self.dateexternal_lastlog = "DATEEXTERNAL_LASTLOG"
# store the fade and real path map
self.mapfadereal = {}
self.initdata()
self.logfolder = logfolder
if self.logfolder == None or self.logfolder == "":
self.logfolder = self.getLogPath()
self.fullfilepaths = []
self.fullfolderpaths = []
self.cp = ConfigParser("configs/config.xml")
self.workpath()
def initdata(self):
self.mapfadereal[self.dateinternal] = ""
self.mapfadereal[self.dateinternal_lastlog] = ""
self.mapfadereal[self.dateexternal] = ""
self.mapfadereal[self.dateexternal_lastlog] = ""
pass
def printData(self):
print self.mapfadereal[self.dateinternal]
print self.mapfadereal[self.dateinternal_lastlog]
print self.mapfadereal[self.dateexternal]
print self.mapfadereal[self.dateexternal_lastlog]
# return the log path if no basepath passed
def getLogPath(self):
if self.logfolder == None or self.logfolder == "":
f = open(self.storefile, "r")
path = f.readline()
# get current path
return path.strip()
else:
return self.logfolder
# return report folder
def getReportPath(self):
logpath = self.getLogPath()
if isLinux():
result = logpath.split("/")
else:
result = logpath.split("\\")
path = result[len(result) - 1]
if path == "":
path = result[len(result) - 2]
if path == "":
path = "xx_unknow_"
# get current path
os.path.abspath('.')
if isLinux():
return os.path.abspath('.') + "/report/" + path.split("_")[1]
else:
return os.path.abspath('.') + "\\report\\" + path.split("_")[1]
def workpath(self):
for parent, dirnames, filenames in os.walk(
self.logfolder): # 三个参数：分别返回1.父目录 2.所有文件夹名字（不含路径） 3.所有文件名字
for dirname in dirnames: # 输出文件夹信息
# print "parent is:" + parent
# check if endwith external_storage
# check if endwith external_storage
# print "dirname is" + dirname
# internal_storage
# internal_storage/last_log
# internal_storage/2014-06-03-19-27-04
if parent.endswith('external_storage'):
if dirname != 'last_log' and dirname.find("-") > 0:
self.mapfadereal[self.dateexternal] = dirname
elif parent.endswith('last_log'):
if parent.find("external_storage") > 0:
self.mapfadereal[self.dateexternal_lastlog] = dirname
elif parent.find("internal_storage") > 0:
self.mapfadereal[self.dateinternal_lastlog] = dirname
pass
if parent.endswith('internal_storage'):
if dirname != 'last_log' and dirname.find("-") > 0:
self.mapfadereal[self.dateinternal] = dirname
self.fullfolderpaths.append(os.path.join(parent, dirname))
#self.printData()
for filename in filenames: # 输出文件信息
# print "parent is"+ parent
# print "filename is:" + filename
# print "the full name of the file is:" + os.path.join(parent,filename) #输出文件路径信息
self.fullfilepaths.append(os.path.join(parent, filename))
def getmap(self):
return self.mapfadereal
def getfiles(self):
return self.fullfilepaths
def getfilesToStr(self):
return ",".join(self.fullfilepaths)
def getfolder(self):
return self.fullfolderpaths
def getfolderToStr(self):
return ",".join(self.fullfolderpaths)
def printmap(self):
print self.mapfadereal
def printfiles(self):
print self.fullfilepaths
def printdirs(self):
print self.fullfolderpaths
def getFileCount(self):
return len(self.fullfilepaths)
def getFolderCount(self):
return len(self.fullfolderpaths)
pass
def getAnrFiles(self):
typename = "anr"
self.getFilesBy(typename)
pass
# return the string list of files with "," split
def getFilesBy(self, typeName):
files = self.cp.getProblemFiles(typeName)
tmprefiles = []
#print files
# get all path meeted files
for i in self.fullfilepaths:
for f in files:
#print "xx"+f.getPath()
#print "yy"+self.getRealPath(f.getPath())
pp = ""
if not isLinux():
pp = f.getPath().replace("/", "\\")
else:
pp = f.getPath().replace("\\", "/")
#print self.getRealPath(pp)
#print i
if i.find(self.getRealPath(pp)) >= 0:
# check items
chk = f.getCheckitem()
#print "chk" + str(chk)
if chk == "" or chk == None:
tmprefiles.append(i)
else:
for ii in chk.split(","):
if i.find(ii) >= 0:
tmprefiles.append(i)
break
break
return ",".join(tmprefiles)
pass
def __checkComplete(self, f):
realPath = f
if realPath.find(self.dateinternal) >= 0:
return None
if realPath.find(self.dateinternal_lastlog) >= 0:
return None
if realPath.find(self.dateexternal) >= 0:
return None
if realPath.find(self.dateexternal_lastlog) >= 0:
return None
'''
if isLinux():
if f.find("/")==0:
pass
else:
f="/"+f
if self.logfolder[len(self.logfolder)-2:len(self.logfolder)-1]=="/":
f=self.logfolder[0:len(self.logfolder)-2]+f
else:
f=self.logfolder+f
else:
if f.find("\\")==0:
pass
else:
f="\\"+f
if self.logfolder[len(self.logfolder)-2:len(self.logfolder)-1]=="\\":
f=self.logfolder[0:len(self.logfolder)-2]+f
else:
f=self.logfolder+f
'''
return f
#return os.path.join(self.logfolder, realPath)
# return null if file path is not exist
def getRealPath(self, fadepath):
result = fadepath
if result == "" or result == None:
return ""
for (k, v) in self.mapfadereal.items():
# print k,v
if v == "":
result = result.replace(k, "nofolderfound")
else:
result = result.replace(k, v)
# print result
pass
return self.__checkComplete(result)
pass
