def setModuleMappings(applicationModel, appFile):
mappings = mapModulesToServers(applicationModel, appFile)
options = ["-MapModulesToServers", mappings]
log(
DEBUG_, "invoking: WebSphere.AdminApp edit " + applicationModel.name +
" " + ` options `)
try:
_excp_ = 0
response = WebSphere.AdminApp.edit(applicationModel.name, options)
except:
_type_, _value_, _tbck_ = sys.exc_info()
_excp_ = 1
#endTry
temp = _excp_
if temp > 0:
log(
ERROR_,
"setModuleMappings: Exception trying to WebSphere.AdminApp edit " +
applicationModel.name + " " + ` options `)
return
#endIf
if len(response) > 0:
log(MAJOR_,
"setModuleMappings: MapModulesToServers response=" + response)
#endIf
log(VERBOSE_, "setModuleMappings: DONE.")
def PreValidateApplicationsExist( applicationModels ):
for applicationModel in applicationModels:
appExists = checkIfAppExists(applicationModel)
if appExists:
log(INFO_, "PreValidateApplicationsExist OK applicationModel=" + `applicationModel`)
else:
fail("PreValidateApplicationsPresent: MISSING application=" + applicationModel.name)
def readWebPage( webpageURL ):
webpageURL = webpageURL.strip()
log(VERBOSE_, "readWebpage webpageURL=" + webpageURL)
url = URL(webpageURL)
conn = url.openConnection()
conn.setConnectTimeout(30000)
conn.setReadTimeout(10000)
conn.connect()
responseCode = conn.getResponseCode()
cookie = conn.getHeaderField("Set-Cookie")
cookiePath = None
pathDiscr = " Path="
if cookie and cookie.find(pathDiscr) > 0:
cookiePath = cookie[cookie.index(pathDiscr) + len(pathDiscr):]
respLines = []
if responseCode >= 400:
log(ERROR_, "HTTP ERROR " + `responseCode` + ": " + `conn.getResponseMessage()`)
else:
log(VERBOSE_, "WebPageResponse status=" + `responseCode` + " reason=" + `conn.getResponseMessage()`)
#log(DEBUG_,"WebPageResponse resp="+`resp` )
reader = BufferedReader(InputStreamReader(conn.getInputStream()))
inputLine = reader.readLine()
while inputLine is not None:
respLines.append(inputLine)
inputLine = reader.readLine()
reader.close()
return respLines, cookiePath
def parse_items(self,response):
"""
解析网页数据
:param response:返回请求成功的HTML
:return: 下一页连接重新封装成Request对象放入待抓取队列，解析出的数据持久化到MongoDB
"""
html = etree.HTML(response["response"])
city = response["city"]
item_data = html.xpath('//div[@class="zu-itemmod "]')
next_item = html.xpath('//div[@class="multi-page"]/a')
if next_item:
if "下一页" in next_item[-1].xpath('text()')[0]:
next_url = next_item[-1].xpath('@href')
if next_url:
log.log("下一页链接 "+next_url[0],"debug")
next_task =Sin_Req(next_url[0],self.parse_items,meta={"city":city})
task_queue.add_task(next_task)
if item_data:
for i in item_data:
item = dict()
item["city"] = city
item["title"] = i.xpath('a/@title')
item["link"] = i.xpath('a/@href')
item["size_style"] = i.xpath('div[@class="zu-info"]/p[@class="details-item tag"]/span[1]/text()')
item["price"] = i.xpath('div[@class="zu-side"]/p/strong/text()')
item["name"] =i.xpath('div[@class="zu-info"]/address/a/text()')
item["address"] = i.xpath('div[@class="zu-info"]/address/text()')
log.log(str(item),"info")
mon.insert("ajk_zufang",item)
def parse_zones(self, response):
city_name = response["city_name"]
base_link = response["city_link"]
html = etree.HTML(response["response"])
items = html.xpath('//ul[@class="search-area-detail clearfix"]/li')[1:]
log.log("ZONE " + str(items), "debug")
for i in items:
zone_ = i.xpath('a/text()')
link_ = i.xpath('a/@href')
if zone_:
data = {}
data["zone_name"] = zone_[0]
data["zone_link"] = link_[0]
log.log(str(data), "info")
task_queue.add_task(
Sin_Req("https:" + urljoin(base_link, link_[0]),
self.parse_details,
meta={
"city_name":
city_name,
"zone_name":
zone_[0],
"zone_base_url":
"https:" + urljoin(base_link, link_[0])
}))
def showAttribute(objName, objType, attName, appName):
try:
if "Application" == objType:
objID = WebSphere.AdminConfig.getid("/Deployment:" + objName + "/")
objID = WebSphere.AdminConfig.showAttribute(objID, "deployedObject")
else:
objID = getModuleID(appName, objName)
#endElse
if attName == "ALL" or attName == "all" or attName == "*" or attName == "":
att = WebSphere.AdminConfig.show(objID)
else:
attNameSplit = attName.split('.')
for attNamePrefix in attNameSplit[:-1]:
if attNamePrefix[0].isupper():
configObjs = WebSphere.AdminConfig.list(attNamePrefix, objID).splitlines()
if len(configObjs) > 0:
objID = configObjs[0]
else:
return None
else:
objID = WebSphere.AdminConfig.showAttribute(objID, attNamePrefix)
attName = attNameSplit[-1]
att = WebSphere.AdminConfig.showAttribute(objID, attName)
#endElse
log(VERBOSE_, "showAtribute: " + objName + " " + attName + "=" + att)
return att
except:
return None
def validateEAR( appPath ):
if not os.path.isfile(appPath):
fail("File does not exist: %s" % appPath)
return 0
else:
return 1
log(VERBOSE_, "validateEAR: FUTURE: installed EAR-FILE validation")
def parse_user_detail(self, response):
try:
data = json.loads(response["response"])
mon.insert("user_detail", data)
log.log(str(data), "info")
except Exception as e:
log.log(str(e), "error")
def validateEAR(appPath):
if not os.path.isfile(appPath):
fail("File does not exist: %s" % appPath)
return 0
else:
return 1
log(VERBOSE_, "validateEAR: FUTURE: installed EAR-FILE validation")
def parse_kb_detail(self, response):
brand = response["brand"]
series = response["series"]
page = response["page"]
base_link = response["base_link"]
text = response["response"]
try:
if len(text) > 50:
data = {}
data["brand"] = brand
data["series"] = series
data["kb_raw"] = text
log.log(str(data), "info")
mon.insert("kb_detail", data)
task_queue.add_task(
Aio_Req(base_link.format(page + 1),
self.parse_kb_detail,
meta={
"base_link": base_link,
"brand": brand,
"series": series,
"page": page + 1
}))
except:
log.log(brand + series + base_link + str(page), "error")
def showAttribute(objName, objType, attName, appName):
try:
if "Application" == objType:
objID = WebSphere.AdminConfig.getid("/Deployment:" + objName + "/")
objID = WebSphere.AdminConfig.showAttribute(
objID, "deployedObject")
else:
objID = getModuleID(appName, objName)
#endElse
if attName == "ALL" or attName == "all" or attName == "*" or attName == "":
att = WebSphere.AdminConfig.show(objID)
else:
attNameSplit = attName.split('.')
for attNamePrefix in attNameSplit[:-1]:
if attNamePrefix[0].isupper():
configObjs = WebSphere.AdminConfig.list(
attNamePrefix, objID).splitlines()
if len(configObjs) > 0:
objID = configObjs[0]
else:
return None
else:
objID = WebSphere.AdminConfig.showAttribute(
objID, attNamePrefix)
attName = attNameSplit[-1]
att = WebSphere.AdminConfig.showAttribute(objID, attName)
#endElse
log(VERBOSE_, "showAtribute: " + objName + " " + attName + "=" + att)
return att
except:
return None
def PreValidateApplicationsAbsent( applicationModels ):
for applicationModel in applicationModels:
appExists = checkIfAppExists(applicationModel)
if appExists:
fail("PreValidateApplicationsAbsent: EXISTING application=" + applicationModel.name)
else:
log(INFO_, "PreValidateApplicationsAbsent: OK applicationModel=" + `applicationModel`)
def parse_items(self,response):
html = etree.HTML(response["response"])
city = response["city"]
item_data = html.xpath('//div[@_soj="xqlb"]')
next_item = html.xpath('//div[@class="multi-page"]/a')
if next_item:
if "下一页" in next_item[-1].xpath('text()')[0]:
next_url = next_item[-1].xpath('@href')
if next_url:
log.log("下一页链接 "+next_url[0],"debug")
next_task =Sin_Req(next_url[0],self.parse_items,meta={"city":city})
task_queue.add_task(next_task)
if item_data:
for i in item_data:
item = dict()
item["city"] = city
item["title"] = i.xpath('a/@title')
item["link"] = i.xpath('a/@href')
item["address"] = i.xpath('div[@class="li-info"]/address/text()')
item["year"] = i.xpath('div[@class="li-info"]/p[@class="date"]/text()')
item["price"] = i.xpath('div[@class="li-side"]/p[1]/strong/text()')
if item["link"]:
commid = re.findall(r'view/(.*?)$',item["link"][0])
if commid:
comm_link = self.base_comm_link+commid[0]
task_queue.add_task(Sin_Req(comm_link,self.parse_comm_trend,meta={"commid":commid}))
log.log(str(item),"info")
mon.insert("ajk_xiaoqu",item)
def parse_brand(self, response):
"""
解析得到汽车品牌的基本信息
:param response:
:return:
"""
try:
data = json.loads(response["response"])
for i in data["data"]["option"]:
log.log(str(i), "info")
mon.insert("car_info", i)
car_name = i["text"]
page = 1
task_link = self.base_items_link.format(series=i["tag_url"],
page=page)
# log.log(task_link,"debug")
item_task = Sin_Req(task_link,
self.parse_item,
meta={
"cur_link": task_link,
"car_name": car_name,
"page": page,
"series": i["tag_url"]
})
task_queue.add_task(item_task)
except Exception as e:
log.log(str(e), "error")
def getPartialClusterNodeServerPairs(clusters):
nodeServerPairs = []
for cluster in clusters:
if len(cluster.inactiveServers) > 0:
cluster_id = WebSphere.AdminConfig.getid("/ServerCluster:" +
cluster.clusterName + "/")
members = WebSphere.AdminConfig.list("ClusterMember", cluster_id)
members = wsadminToList(members)
for member in members:
node = WebSphere.AdminConfig.showAttribute(member, "nodeName")
server = WebSphere.AdminConfig.showAttribute(
member, "memberName")
log(
DEBUG_,
"getNodeServerPairs: cluster=" + cluster.clusterName +
" contains node=" + node + " server=" + server)
inactive = 0
for inactiveServer in cluster.inactiveServers:
if (node, server) == (inactiveServer.nodeName,
inactiveServer.serverName):
inactive = 1
if not inactive:
nodeServerPair = ServerApplicationTarget(
node, server, cluster.tests, cluster.jvmAttributes)
nodeServerPairs.append(nodeServerPair)
#endFor
#endFor
log(DEBUG_,
"getNodeServerPairs: returning nodeServerPairs=" + ` nodeServerPairs `)
return nodeServerPairs
def parse_detail(self, response):
"""
解析检测报告详情页
:param response:
:return:
"""
car_id = response["data_id"]
html = etree.HTML(response["response"])
items = html.xpath('//div[@class="other clearfixnew"]/div')
for i in items:
for j in i.xpath('div[@class="option"]'):
data = dict()
data["car_id"] = car_id
option_name = j.xpath('div[@class="child-title"]/text()')
option_counts = j.xpath('div[@class="mun"]/text()')
data["option_name"] = option_name
data["option_counts"] = option_counts
if j.xpath('a[@class="test-fail-box"]'):
fetal_counts = j.xpath(
'a[@class="test-fail-box"]/div[@class="test-fail"]/div[@class="mun"]/text()'
)
data["fetal_counts"] = fetal_counts
else:
fetal_counts = 0
data["fetal_counts"] = fetal_counts
mon.insert("report_data", data)
log.log(str(data), "info")
async def on_message(message):
try:
if message.content.strip().lower().startswith(PREFIX + "usage") or message.content.lower().startswith("/help"):
await message.channel.send("""Hi there, you can find the working functionality:
- `""" + PREFIX + """r` : roll some dices using the format: `1d20+10`
- `""" + PREFIX + """rf` : roll a D&D/Pathfinder full attack! Use the format: `1d20+15/+10/+5` WORK IN PROGRESS
- `""" + PREFIX + """git` : get the GitHub project link
If something does not work please open an issue on GitHub: https://github.com/Irekean/DiRer_DiceRoller/issues
""")
elif message.content.strip().lower().startswith(PREFIX + 'rf '):
await message.channel.send(roll_full_attack(message))
elif message.content.strip().lower().startswith(PREFIX + 'r ') or message.content.strip().lower().startswith(
PREFIX + 'roll '):
await message.channel.send(normal_roll(message))
elif (message.content.lower().strip() + " ").startswith(PREFIX + "git "):
await message.channel.send("The bot code is on GitHub: https://github.com/Irekean/DiRer_DiceRoller")
elif message.author.id == 322449846336356363 and (
message.content.strip().lower().startswith(PREFIX + 'kill') or message.content.strip().lower().startswith(
PREFIX + 'stop')):
await client.logout()
elif message.content.strip().startswith(PREFIX): # Always keep as last
await message.channel.send("Need help? Try with `" + PREFIX + "help` or `" + PREFIX + "usage`")
except:
log.log("Unexpected error:" + sys.exc_info()[0])
raise
def addExistingTargets(applicationName, applicationTarget, applicationClusters, applicationServers):
deployment = WebSphere.AdminConfig.getid("/Deployment:" + applicationName + "/")
deployedObject = WebSphere.AdminConfig.showAttribute(deployment, "deployedObject")
modules = wsadminToList(WebSphere.AdminConfig.showAttribute(deployedObject, "modules"))
clusters = []
serverNodes = []
for module in modules:
targetMappings = wsadminToList(WebSphere.AdminConfig.showAttribute(module, "targetMappings"))
for targetMapping in targetMappings:
target = WebSphere.AdminConfig.showAttribute(targetMapping, "target")
if re.search("#ClusteredTarget", target):
clusterName = WebSphere.AdminConfig.showAttribute(target, "name")
if clusterName not in clusters:
log(INFO_, "existing clusterName: " + clusterName)
clusters.append(clusterName)
elif re.search("#ServerTarget", target):
serverName = WebSphere.AdminConfig.showAttribute(target, "name")
nodeName = WebSphere.AdminConfig.showAttribute(target, "nodeName")
if [nodeName, serverName] not in serverNodes:
log(INFO_, "existing node server: %s:%s" % (nodeName, serverName))
serverNodes.append([nodeName, serverName])
for serverNode in serverNodes:
if getServerType(serverNode[0], serverNode[1]) == "APPLICATION_SERVER":
applicationServers.append(ServerApplicationTarget(serverNode[0], serverNode[1], applicationTarget.tests,
applicationTarget.jvmAttributes))
for clusterName in clusters:
clusterTarget = ClusterApplicationTarget(clusterName, applicationTarget.tests, applicationTarget.jvmAttributes)
clusterTarget.inactiveServers = getInactiveServers(clusterTarget)
applicationClusters.append(clusterTarget)
def execWithLog(command,
argv,
root='/',
level=logging.INFO,
timeoutInSecs=0,
raiseException=False):
'''Execute the given command and log its output.'''
cmdline = " ".join(argv)
log.debug("executing: %s" % cmdline)
(output, status) = execWithCapture(command,
argv,
root=root,
returnStatus=True,
catchfdList=[STDOUT, STDERR],
timeoutInSecs=timeoutInSecs,
raiseException=False)
log.debug("command exited with status %d" % status)
if status:
log.error(output)
if raiseException:
raise ExecError(cmdline, output, status)
else:
log.log(level, output)
return status
def get_one_url(self):
while True:
url, callback, retries = self._wait_urls.get_nowait()
if retries < 3:
break
else:
log.log("%s\tfailed" % (url))
return (url, callback, retries)
def get_one_url(self):
while True:
url, callback, retries = self._wait_urls.get_nowait()
if retries < 3:
break
else:
log.log("%s\tfailed"%(url))
return (url, callback, retries)
def get(self):
try:
r = requests.get(url=self.url, stream=True)
log.log(str(r.status_code) + " " + self.url, "info")
self.meta["response"] = r.content
self.callback(self.meta)
except Exception as e:
# tt.add_task(Sin_Req(self.url, self.callback,meta=self.meta))
log.log(self.url + " " + str(e), "error")
def run(self, nums):
while True:
task_ = [task_queue.pop_task() for _ in range(nums)]
tasks = [i.aio_req() for i in task_ if i != None]
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
if None in task_:
log.log("任务队列已空！", "info")
break
def call_onmatch_metod(self, *args, **kwargs):
try:
return getattr(self, self.onmatch_method_name, '')(*args, **kwargs)
except AttributeError:
log.log('error', 'wrong onmatch (AttributeError)')
return None
except TypeError:
log.log('error', 'wrong onmatch (TypeError)')
return None
def checkServerStarted(nodeName, serverName):
log(
VERBOSE_, "checkServerStarted: nodeName=" + nodeName + " serverName=" +
serverName + " ...")
serverID = ""
retries = 0
try:
serverID = WebSphere.AdminControl.completeObjectName("node=" +
nodeName +
",name=" +
serverName +
",type=Server,*")
except:
_type_, _value_, _tbck_ = sys.exc_info()
log(
WARNING_, "checkServerStarted WebSphere.AdminControl exception=" +
` _type_ ` + " serverID=" + serverID)
while len(serverID) == 0 and retries <= 15:
log(
VERBOSE_, "checkServerStarted: server not yet started, retries=" +
` retries ` + " ...")
sleepDelay(5 + 2 * retries)
retries = retries + 1
try:
serverID = WebSphere.AdminControl.completeObjectName(
"node=" + nodeName + ",name=" + serverName + ",type=Server,*")
except:
_type_, _value_, _tbck_ = sys.exc_info()
log(
WARNING_,
"checkServerStarted WebSphere.AdminControl exception=" +
` _type_ ` + " serverID=" + serverID)
#endIf
#endWhile
if retries > 0:
log(
INFO_, "checkServerStarted: " + nodeName + " " + serverName +
" had slow start, status retries=" + ` retries `)
#endIf
if serverID == "":
log(
ERROR_, "checkServerStarted: " + nodeName + " " + serverName +
" server FAILED TO START.")
return
#endIf
state = WebSphere.AdminControl.getAttribute(serverID, "state")
state = state.upper()
if state == "STARTED":
log(
INFO_, "checkServerStarted: " + nodeName + " " + serverName +
" state=" + state)
else:
log(
ERROR_, "checkServerStarted: " + nodeName + " " + serverName +
" INCORRECT state=" + state)
#endElse
log(VERBOSE_, "checkServerStarted: DONE.")
def get(self):
try:
r = requests.post(url=self.url,data = self.post_data,headers = self.headers,proxies = PROXIES)
log.log(str(r.status_code)+" "+self.url,"info")
self.meta["response"] = r.text
self.callback(self.meta)
# tt.old_task(self.url)
except Exception as e:
tt.add_task(Post_Req(self.url,self.post_data,self.callback,meta=self.meta))
log.log(self.url+" "+str(e),"error")
def PreValidateNodesAndServers( uniqueNodesContainedServers ):
if not len(uniqueNodesContainedServers):
log(WARNING_, "PreValidateNodesAndServers: No nodes/servers/clusters specified")
#endIf
for nodeContainedServers in uniqueNodesContainedServers:
nodeName = nodeContainedServers[0]
PreValidateNode(nodeName)
containedServers = nodeContainedServers[1]
for serverName in containedServers:
PreValidateServer(nodeName, serverName)
def get_authorized_connection(host, password, port=30004):
log.log('debug', 'connecting...')
connection = Telnet(host, port)
connection.read_until('password:'******'pass\n')
connection.read_until('reenter password:'******'\n')
connection.read_until("\r\n= \r\n\r\n")
log.log('error', 'connection estabilised')
return connection
def parse_report(self, response):
log.log("检测报告", "debug")
car_id = response["car_id"]
raw_data = response["response"].encode("utf-8").decode(
"unicode_escape")
report_data = json.loads(raw_data)
data = {}
data[car_id[0]] = report_data
log.log("report data " + str(data), "info")
mon.insert("report_data", data)
def determineUniqueNodesContainedServers(nodeServerPairs):
log(
DEBUG_, "determineUniqueNodesContainedServers: nodeServerPairs=" +
` nodeServerPairs `)
nodesContainedServers = []
for nodeServer in nodeServerPairs:
nodeName = nodeServer.nodeName
serverName = nodeServer.serverName
nodeIndex = 0
for uniquenodeContainedServers in nodesContainedServers:
uniquenode = uniquenodeContainedServers[0]
if uniquenode == nodeName:
containedServers = uniquenodeContainedServers[1]
for server in containedServers:
if server == serverName:
serverName = ""
break
#endIf
#endFor
if serverName != "":
containedServers.append(serverName)
nodeContainedServers = [nodeName, containedServers]
log(
DEBUG_,
"determineUniqueNodesContainedServers: Replacing node="
+ nodeName + " with NEW containedServers=" +
` containedServers `)
del nodesContainedServers[nodeIndex]
nodesContainedServers.insert(nodeIndex,
nodeContainedServers)
log(
DEBUG_,
"determineUniqueNodesContainedServers: New nodesContainedServers="
+ ` nodesContainedServers `)
#endIf
nodeName = ""
break
#endIf
nodeIndex = nodeIndex + 1
#endFor
if nodeName != "" and serverName != "":
nodeContainedServers = [nodeName, [serverName]]
log(
DEBUG_,
"################## determineUniqueNodesContainedServers: new node="
+ nodeName + " new server=" + serverName)
nodesContainedServers.append(nodeContainedServers)
#endIf
log(DEBUG_, "nodesContainedServers=" + ` nodesContainedServers `)
#endFor
log(
DEBUG_,
"determineUniqueNodesContainedServers: returning LIST nodesContainedServers="
+ ` nodesContainedServers `)
return nodesContainedServers
def saveAndDisableAutosync(action, nodesContainedServers):
if action == "update":
pass
else:
fail("saveAndDisableAutosync: should never be called for action=" +
action + " (only for 'install')")
#endElse
log(INFO_, "")
log(MAJOR_, "saveAndDisableAutosync of affectedNodes begin ...")
savedNodesAutosyncs = []
for nodeContainedServer in nodesContainedServers:
nodeName = nodeContainedServer[0]
nodeAgent = WebSphere.AdminConfig.getid("/Node:" + nodeName +
"/Server:nodeagent/")
syncServ = WebSphere.AdminConfig.list("ConfigSynchronizationService",
nodeAgent)
syncEnabled = WebSphere.AdminConfig.showAttribute(
syncServ, "autoSynchEnabled")
synchOnServerStartup = WebSphere.AdminConfig.showAttribute(
syncServ, "synchOnServerStartup")
log(
INFO_,
"nodeContainedServers: nodeName=" + nodeName + " syncEnabled=" +
syncEnabled + " synchOnServerStartup=" + synchOnServerStartup)
if syncEnabled:
log(
MAJOR_,
"saveAndDisableAutosync: temporarily setting AutoSyncEnabled FALSE for node="
+ nodeName)
WebSphere.AdminConfig.modify(syncServ,
[["autoSynchEnabled", "false"]])
#endIf
if synchOnServerStartup:
log(
MAJOR_,
"saveAndDisableAutosync: temporarily setting SynchOnServerStartup FALSE for node="
+ nodeName)
WebSphere.AdminConfig.modify(syncServ,
[["synchOnServerStartup", "false"]])
#endIf
savedNodeAutosync = [nodeName, syncEnabled, synchOnServerStartup]
log(
DEBUG_, "saveAndDisableAutosync: nodeName=" + nodeName +
" savedNodeAutosync=" + ` savedNodeAutosync `)
savedNodesAutosyncs.append(savedNodeAutosync)
#endFor
configSave()
log(
DEBUG_, "saveAndDisableAutosync: returning node savedNodesAutosyncs=" +
` savedNodesAutosyncs `)
highlight(MAJOR_, "saveAndDisableAutosync of affectedNodes DONE.")
return savedNodesAutosyncs
def PreValidateNodesAndServers(uniqueNodesContainedServers):
if not len(uniqueNodesContainedServers):
log(WARNING_,
"PreValidateNodesAndServers: No nodes/servers/clusters specified")
#endIf
for nodeContainedServers in uniqueNodesContainedServers:
nodeName = nodeContainedServers[0]
PreValidateNode(nodeName)
containedServers = nodeContainedServers[1]
for serverName in containedServers:
PreValidateServer(nodeName, serverName)
def execScript(applicationModel, scriptName):
if applicationModel.scripts.has_key(scriptName):
try:
log(INFO_,
"executing %s for %s: '%s'" % (scriptName, applicationModel.name, applicationModel.scripts[scriptName]))
print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
rc = javaos.system(applicationModel.scripts[scriptName])
print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
return rc
except:
return -1
def run(self, nums):
while True:
task_ = [task_queue.pop_task() for _ in range(nums)]
tasks = [th.Thread(target=i.get()) for i in task_ if i != None]
for i in tasks:
i.start()
for i in tasks:
i.join()
if None in task_:
log.log("任务队列已空！", "info")
break
def moss_query_runner():
info = moss_queue.get()
while info != -1:
query_helper = MossInteractor(*info)
log.log(f'Processing query for problem {info.problem_id}')
query_helper.send_moss_query()
if query_helper.report_exists():
query_helper.parse_report()
else:
log.log(f'Report for problem {info.problem_id} not found')
info = moss_queue.get()
def parse_user_detail(self,response):
"""
解析用户的详细信息持久化到数据库
:param response:
:return:
"""
try:
data = json.loads(response["response"])
mon.insert("user_detail",data)
log.log(str(data),"info")
except Exception as e:
log.log(str(e),"error")
def StopApplicationOnNodesAndServers( applicationModel, uniqueNodeServerPairs ):
log(MAJOR_,
"StopApplicationOnNodesAndServers: applicationModel=" + applicationModel.name + " nodeServerPairs=" + `uniqueNodeServerPairs` + "...")
if not len(uniqueNodeServerPairs):
fail("StopApplicationOnNodesAndServers : No nodes/servers/clusters specified")
#endIf
for nodeServer in uniqueNodeServerPairs:
nodeName = nodeServer.nodeName
serverName = nodeServer.serverName
StopApplication(applicationModel, nodeName, serverName)
#endFor
highlight(MAJOR_, "StopApplicationOnNodesAndServers DONE.")
def spreadsheet_CreateAForm(self):
"""
新建测试报告电子表格文件
:return:
"""
print("开始新建测试报告表", __file__, sys._getframe().f_lineno)
log.log("开始新建测试报告表", __file__, sys._getframe().f_lineno)
style = xlwt.XFStyle() # 创建样式
self.sheet_name = "测试报告"
workbook = xlwt.Workbook(encoding='utf-8') # 新建工作簿
sheet = workbook.add_sheet(self.sheet_name) # 新建sheet
# 设置单元格高
tall_style = xlwt.easyxf('font:height 360')
first_row = sheet.row(0)
first_row.set_style(tall_style)
first_row1 = sheet.row(1)
first_row1.set_style(tall_style)
# 样式：居中
al = CellStyle().spreadsheet_CreateAForm_style_alignCenter()
# 样式：添加边框
borders = CellStyle().spreadsheet_CreateAForm_style_rim()
# 把样式和边框导给样式style
style.borders = borders
style.alignment = al
# 设置列表的宽
sheet.col(0).width = 256 * 30
sheet.col(1).width = 256 * 20
sheet.col(2).width = 256 * 30
sheet.col(3).width = 256 * 30
sheet.col(4).width = 256 * 30
sheet.col(5).width = 256 * 30
t = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
sheet.write_merge(0, 0, 0, 5, '%r测试报告' % t, style) # 输入标题
# 输入副标题
sheet.write(1, 0, "所属接口", style)
sheet.write(1, 1, "所属模块", style)
sheet.write(1, 2, " 测试点", style)
sheet.write(1, 3, "预期结果", style)
sheet.write(1, 4, "实际结果", style)
sheet.write(1, 5, "测试结果", style)
relativeAddress = relative_site.location() # 获取项目相对位置
# 获取测试的系统
systematicName = ProfileDataProcessing("commonality",
"system").config_File()
# 拼接测试模板位置
fileAddress = relativeAddress + "report" + "/" + systematicName + "/" + self.table_name
print("fileAddress", fileAddress)
workbook.save(fileAddress)
# 返回表单名和测试报告名称用以插入数据
print("测试报告表新建完成", __file__, sys._getframe().f_lineno)
log.log("开始新建测试报告表", __file__, sys._getframe().f_lineno)
return self.sheet_name, self.table_name
async def aio_req(self):
async with aiohttp.ClientSession() as resp:
try:
async with resp.get(url=self.url,headers=self.headers) as resp:
page = await resp.text()
self.meta["response"] = page
log.log(str(resp.status) + " " + self.url, "info")
self.callback(self.meta)
tt.old_task(self.url)
except Exception as e:
print(e)
tt.add_task(Aio_Req(self.url,self.callback,meta=self.meta))
log.log(self.url + " " + str(e),"error")
def StartApplicationOnNodesAndServers( applicationModel, uniqueNodeServerPairs ):
log(INFO_, "")
log(MAJOR_,
"StartApplicationOnNodesAndServers: applicationModel=" + applicationModel.name + " nodeServerPairs=" + `uniqueNodeServerPairs` + "...")
if not len(uniqueNodeServerPairs):
fail("StartApplicationOnNodesAndServers : No nodes/servers/clusters specified")
#endIf
for nodeServer in uniqueNodeServerPairs:
nodeName = nodeServer.nodeName
serverName = nodeServer.serverName
StartApplication(applicationModel, nodeName, serverName)
#endFor
highlight(MAJOR_, "StartApplicationOnNodesAndServers DONE.")
def execScript(applicationModel, scriptName):
if applicationModel.scripts.has_key(scriptName):
try:
log(
INFO_, "executing %s for %s: '%s'" %
(scriptName, applicationModel.name,
applicationModel.scripts[scriptName]))
print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
rc = javaos.system(applicationModel.scripts[scriptName])
print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
return rc
except:
return -1
def uninstall(action, distDir, wasRoot):
log(VERBOSE_, "uninstall: " + action + " " + distDir)
log(VERBOSE_, "uninstall: " + wasRoot + " ...")
############### FIND APPLICATIONS ####################
applicationModels = getApplications(distDir)
for applicationModel in applicationModels:
log(INFO_, "uninstall: Deployment applicationModel=" + `applicationModel.name`)
################## PRE-VALIDATE APPLICATIONS (exists) ####################
PreValidateApplicationsExist(applicationModels)
############### CALCULATE AFFECTED NODES ####################
calculateAffectedNodes(action, wasRoot, applicationModels)
################## PRE-VALIDATE NODES and SERVERS ####################
PreValidateNodesAndServers(Globals.uniqueNodesContainedServers)
ValidateSynched(Globals.uniqueNodesContainedServers)
listApplications()
#endFor
for item in Globals.appsNodesServers:
applicationModel = item[0]
appExists = checkIfAppExists(applicationModel)
if appExists:
StopApplicationOnNodesAndServers(applicationModel, item[1])
uninstallEAR(applicationModel)
#endIf
#endFor
listApplications()
configSave()
for applicationModel in applicationModels:
log(INFO_, "DONE: uninstall application=" + `applicationModel.name`)
def getInactiveServers( cluster ):
inactiveServers = []
cluster_id = WebSphere.AdminConfig.getid("/ServerCluster:" + cluster.clusterName + "/")
members = WebSphere.AdminConfig.list("ClusterMember", cluster_id)
members = wsadminToList(members)
for member in members:
nodeName = WebSphere.AdminConfig.showAttribute(member, "nodeName")
serverName = WebSphere.AdminConfig.showAttribute(member, "memberName")
log(DEBUG_,
"getNodeServerPairs: cluster=" + cluster.clusterName + " contains node=" + nodeName + " server=" + serverName)
if not PreValidateServer(nodeName, serverName):
nodeServerPair = ServerApplicationTarget(nodeName, serverName, cluster.tests, cluster.jvmAttributes)
inactiveServers.append(nodeServerPair)
#endFor
return inactiveServers
def deploy(action, failOnError, distDir, wasRoot):
log(VERBOSE_, "deploy: INPUTS: " + action + " " + distDir + " " + wasRoot)
action = action.lower()
if action == "install" or action == "update":
installOrUpdate(action, distDir, wasRoot)
elif action == "confirm":
confirm(action, distDir, wasRoot)
elif action == "syncripplestart":
syncRippleStartDone(distDir, wasRoot)
elif action == "uninstall":
uninstall(action, distDir, wasRoot)
else:
msg = "ERROR: deploy: unknown action=" + action + " (must be 'install' or 'update' or 'confirm' or 'syncRippleStart' or 'uninstall'"
fail(msg)
def getNodeServerPairs( clusters ):
nodeServerPairs = []
for cluster in clusters:
cluster_id = WebSphere.AdminConfig.getid("/ServerCluster:" + cluster.clusterName + "/")
members = WebSphere.AdminConfig.list("ClusterMember", cluster_id)
members = wsadminToList(members)
for member in members:
node = WebSphere.AdminConfig.showAttribute(member, "nodeName")
server = WebSphere.AdminConfig.showAttribute(member, "memberName")
log(DEBUG_,
"getNodeServerPairs: cluster=" + cluster.clusterName + " contains node=" + node + " server=" + server)
nodeServerPair = ServerApplicationTarget(node, server, cluster.tests, cluster.jvmAttributes)
nodeServerPairs.append(nodeServerPair)
#endFor
#endFor
log(DEBUG_, "getNodeServerPairs: returning nodeServerPairs=" + `nodeServerPairs`)
return nodeServerPairs
def _execute(self, script):
from preparser import ScriptedInstallPreparser
from scriptedinstallutil import Result
errors = None
installCompleted = False
try:
self.sip = ScriptedInstallPreparser(script)
(result, errors, warnings) = self.sip.parseAndValidate()
if warnings:
log.warn("\n".join(warnings))
if errors:
log.error("\n".join(errors))
userchoices.setReboot(False)
if result != Result.FAIL:
# Bring up whatever is needed for the install to happen. For
# example, get the network going for non-local installs.
errors, warnings = self._runtimeActions()
if warnings:
log.warn("\n".join(warnings))
if errors:
log.error("\n".join(errors))
userchoices.setReboot(False)
if not errors:
if userchoices.getDebug():
log.info(userchoices.dumpToString())
if userchoices.getDryrun():
log.log(LOGLEVEL_HUMAN, "dry run specified, stopping.")
else:
context = applychoices.Context(
applychoices.ProgressCallback(
applychoices.StdoutProgressDelegate()))
applychoices.doit(context)
installCompleted = True
media.runtimeActionEjectMedia()
except IOError, e:
log.error("error: cannot open file -- %s\n" % str(e))
def syncRippleStart(action, applicationModels):
for nodeContainedServers in Globals.uniqueNodesContainedServers:
nodeName = nodeContainedServers[0]
syncNDNode(nodeName)
log(MAJOR_, "syncRippleStart synced: " + nodeName)
for applicationModel in applicationModels:
execScript(applicationModel, "afterSync")
for clusterName in Globals.clusters.keys():
cluster = WebSphere.AdminControl.completeObjectName('type=Cluster,name=%s,*' % clusterName)
clusterTarget = Globals.clusters[clusterName]
if not len(clusterTarget.inactiveServers):
WebSphere.AdminControl.invoke(cluster, 'rippleStart')
log(MAJOR_, "syncRippleStart: rippleStart " + clusterName + " invoked")
sleepDelay(3)
for nodeServerPair in Globals.unclusteredNodeServerPairs:
stopNDServer(nodeServerPair[0], nodeServerPair[1])
startNDServer(nodeServerPair[0], nodeServerPair[1])
checkClustersStarted(Globals.clusters)
for applicationModel in applicationModels:
execScript(applicationModel, "afterRippleStart")
def confirm(action, distDir, wasRoot):
log(VERBOSE_, "confirm: " + action + " " + distDir)
log(VERBOSE_, "confirm: " + wasRoot + " ...")
############### FIND APPLICATIONS ####################
applicationModels = getApplications(distDir, "*.xml.done")
applicationModels.extend(getApplications(distDir, "*.xml.confirmed"))
for applicationModel in applicationModels:
log(INFO_, "confirm: Deployment applicationModel=" + `applicationModel.name`)
################## PRE-VALIDATE APPLICATIONS (exists) ####################
PreValidateApplicationsExist(applicationModels)
############### CALCULATE AFFECTED NODES ####################
calculateAffectedNodes(action, wasRoot, applicationModels)
################## PRE-VALIDATE NODES and SERVERS ####################
PreValidateNodesAndServers(Globals.uniqueNodesContainedServers)
################## TEST ####################
testApplication(applicationModels)
nodeSyncStatus = {}
for nodeContainedServers in Globals.uniqueNodesContainedServers:
node = nodeContainedServers[0]
if not node in nodeSyncStatus.keys():
nodeSync = WebSphere.AdminControl.completeObjectName("type=NodeSync,node=%s,*" % node)
nodeSyncStatus[node] = WebSphere.AdminControl.invoke(nodeSync, "isNodeSynchronized")
print "NODE SYNC STATUS: %s" % nodeSyncStatus
highlight(MAJOR_, "confirm: DONE.")
def configSave( ):
log(MAJOR_, "configSave: ...")
saved = WebSphere.AdminConfig.save()
if len(saved) > 0:
log(INFO_, saved)
#endIf
log(VERBOSE_, "configSave: DONE.")
def installEAR( action, appPath, applicationModel, clusterName, nodeName, serverName, installOptions ):
update = "-appname '%s'" % applicationModel.name
if action == "update":
update = "-update " + update
#endIf
if serverName != "" and nodeName != "":
options = update + " -verbose -node " + nodeName + " -server " + serverName + " -distributeApp " + installOptions
options = options + " -MapWebModToVH " + AArrayToOptionString(mapWebModToVH(applicationModel, appPath))
options = options + " -MapModulesToServers " + AArrayToOptionString(
mapModulesToServers(applicationModel, appPath))
highlight(MAJOR_, "AdminApp.install(" + appPath + "," + options + ")")
installed = WebSphere.AdminApp.install(appPath, options)
#endIf
elif clusterName != "":
options = update + " -verbose -cluster " + clusterName + " -distributeApp " + installOptions
options = options + " -MapWebModToVH " + AArrayToOptionString(mapWebModToVH(applicationModel, appPath))
options = options + " -MapModulesToServers " + AArrayToOptionString(
mapModulesToServers(applicationModel, appPath))
highlight(MAJOR_, "AdminApp.install(" + appPath + "," + options + ")")
installed = WebSphere.AdminApp.install(appPath, options)
#endIf
else:
options = update + " -verbose -distributeApp " + installOptions
options = options + " -MapWebModToVH " + AArrayToOptionString(mapWebModToVH(applicationModel, appPath))
options = options + " -MapModulesToServers " + AArrayToOptionString(
mapModulesToServers(applicationModel, appPath))
highlight(MAJOR_, "AdminApp.install(" + appPath + "," + options + ")")
installed = WebSphere.AdminApp.install(appPath, options)
#endElse
if len(installed) > 0:
log(INFO_, installed)
#endIf
appExists = checkIfAppExists(applicationModel)
if appExists:
pass
else:
fail("failed to installEAR application=" + applicationModel.name)
#endElse
log(VERBOSE_, "InstallEAR: DONE.")
def listApplications( ):
log(INFO_, "ListApplications:")
apps = WebSphere.AdminApp.list()
apps = wsadminToList(apps) # running on windows, target is linux (different SEPARATOR)
for app in apps:
log(INFO_, " " + app)
#endFor
log(VERBOSE_, "ListApplications: DONE.")
def PreValidateNode( nodeName ):
nodeID = WebSphere.AdminConfig.getid("/Node:" + nodeName + "/")
if not len(nodeID):
msg = "PreValidateNode: failed for node=" + nodeName + " (invalid nodeName)"
log(WARNING_, msg)
#endIf
ndSync = WebSphere.AdminControl.completeObjectName("type=NodeSync,node=" + nodeName + ",*")
if ndSync == "":
log(WARNING_, "PreValidateNode: failed for nodeName=" + nodeName + " (could not access " + nodeName + ")")
#endIf
nodeAgent = WebSphere.AdminConfig.getid("/Node:" + nodeName + "/Server:nodeagent/")
if nodeAgent == "":
log(WARNING_, "PreValidateNode: failed for nodeName=" + nodeName + " (nodeAgent STOPPED)")
else:
log(VERBOSE_, "PreValidateNode OK nodeName=" + nodeName)
def uninstallEAR( applicationModel ):
log(MAJOR_, "UninstallEAR: " + applicationModel.name + "...")
uninstalled = WebSphere.AdminApp.uninstall(applicationModel.name)
log(INFO_, uninstalled)
appExists = checkIfAppExists(applicationModel)
if appExists:
fail("failed to uninstallEAR application=" + applicationModel.name)
#endIf
log(VERBOSE_, "UninstallEAR: DONE.")
def checkIfAppNameExists( appName ):
appExists = true
application = WebSphere.AdminConfig.getid("/Deployment:" + appName + "/")
log(DEBUG_, "checkIfAppExists applicationModel=" + appName + " installedAppID=" + application)
if not len(application):
appExists = false
log(VERBOSE_, "checkIfAppExists: FALSE for applicationModel=" + appName)
else:
log(VERBOSE_, "checkIfAppExists: TRUE for applicationModel=" + appName)
#endElse
return appExists
def syncNDNode(nodeName):
EarExpandDelay = 10
log(MAJOR_, "syncNDNode: ReSync of ND Node=" + nodeName + " (actual application distribution to Server(s) ) ...")
ndSync = WebSphere.AdminControl.completeObjectName("type=NodeSync,node=" + nodeName + ",*")
if ndSync == "":
fail("cannot syncNDNode (stopped?) nodeName=" + nodeName)
#endIf
sync = WebSphere.AdminControl.invoke(ndSync, "sync")
log(INFO_, "syncNDNode: " + nodeName + " invoked sync=" + sync + " DONE.")
log(VERBOSE_,
"syncNDNode: FUTURE: check for nodeSync EAR expansion complete (for now just delay " + `EarExpandDelay` + " secs)")
sleepDelay(EarExpandDelay)
def stopSyncStart(action, nodeName, containedServerNames):
log(MAJOR_, "stopSyncStart: nodename=" + nodeName + " containedServernames=" + `containedServerNames` + " ...")
syncNDNode(nodeName)
if action == "update":
for serverName in containedServerNames:
stopNDServer(nodeName, serverName)
startNDServer(nodeName, serverName)
#endFor
if action == "update":
log(VERBOSE_,
"stopSyncStart: FUTURE: plugin-cfg.xml RE-ACTIVATE node=" + nodeName + " servers=" + `containedServerNames`)
#endIf
log(MAJOR_, "stopSyncStart: DONE. (nodeName=" + nodeName + ")")
def PreValidateCluster( cluster):
clusterID = WebSphere.AdminConfig.getid("/ServerCluster:" + cluster.clusterName + "/")
if not len(clusterID):
msg = "PreValidateCluster: failed for cluster.clusterName=" + cluster.clusterName + " (invalid cluster.clusterName)"
log(WARNING_, msg)
#endIf
clusterID = WebSphere.AdminControl.completeObjectName("name=" + cluster.clusterName + ",*")
if not len(clusterID):
msg = "PreValidateCluster: cannot access cluster.clusterName=" + cluster.clusterName + " (cluster STOPPED)"
log(WARNING_, msg)
else:
state = WebSphere.AdminControl.getAttribute(clusterID, "state")
log(VERBOSE_, "PreValidateCluster OK cluster.clusterName=" + cluster.clusterName + " state=" + state)
def ValidateSynched( uniqueNodesContainedServers ):
for nodeContainedServers in uniqueNodesContainedServers:
nodeName = nodeContainedServers[0]
ndSync = WebSphere.AdminControl.completeObjectName("type=NodeSync,node=" + nodeName + ",*")
if ndSync == "":
log(WARNING_, "ValidateSynched: failed for nodeName=" + nodeName + " (could not access " + nodeName + ")")
else:
status = AdminControl.invoke(ndSync, "isNodeSynchronized")
if status != true:
log(WARNING_,
"ValidateSynched: failed for nodeName=" + nodeName + " (isNodeSynchronized returned " + status + ")")
else:
log(INFO_, "Node is Synchronized:" + nodeName)
def setModuleMappings( applicationModel, appFile ):
mappings = mapModulesToServers(applicationModel, appFile)
options = ["-MapModulesToServers", mappings]
log(DEBUG_, "invoking: WebSphere.AdminApp edit " + applicationModel.name + " " + `options`)
try:
_excp_ = 0
response = WebSphere.AdminApp.edit(applicationModel.name, options)
except:
_type_, _value_, _tbck_ = sys.exc_info()
_excp_ = 1
#endTry
temp = _excp_
if temp > 0:
log(ERROR_,
"setModuleMappings: Exception trying to WebSphere.AdminApp edit " + applicationModel.name + " " + `options`)
return
#endIf
if len(response) > 0:
log(MAJOR_, "setModuleMappings: MapModulesToServers response=" + response)
#endIf
log(VERBOSE_, "setModuleMappings: DONE.")
