def run(self, context):
self.ts_run_start = time.time()
if not self.skipRun:
# TODO: replace input with self.rrInput
# if self.rrInput:
# pass
klog.d(varfmt(self))
# klog.d(varfmt(context))
klog.d(">>> Run action")
res = self.orgRun(context)
klog.d("<<< Run action")
klog.d(varfmt(res))
try:
_input = todict(self)
except:
_input = None
try:
_output = todict(res)
except:
_output = None
kfk_trace.log(kfk_etypes.TASK_ACTION, self.atom, "RUNNING",
self.workflow_name, self.workflow_execution_id,
self.task_id, self.task_name,
_input, _output, None)
else:
klog.d("YIHE.Sync.run skipped")
# TODO: replace output with self.rrOutput
if self.rrOutput:
pass
self.ts_run_end = time.time()
klog.d("TIME: Init total: ", self.ts_init_end - self.ts_init_start)
klog.d("TIME: Run total: ", self.ts_run_end - self.ts_run_start)
klog.d("TIME: Action Total: ", self.ts_run_end - self.ts_init_start)
if self.ts_ses_end:
klog.d("TIME: Session Total: ", self.ts_ses_end - self.ts_ses_start)
return res
def __init__(self, action_context, cmd, psid=None, sid=None):
actx = action_context
klog.d(" ACTX : " + str(varfmt(actx)))
klog.d(" CMD : " + str(cmd))
klog.d(" PSID : " + str(psid))
klog.d(" SID : " + str(sid))
self.cmd = cmd
self.actx = actx
self.psid = psid
self.sid = sid
def run(self, context):
klog.d("CMD: ", self.cmd)
klog.d(varfmt(context))
if self.cmd == "create":
# Get content information
# action_execution_id = self.actx.get("action_execution_id")
# task_id = self.actx.get("task_id")
# task_name = self.actx.get("task_name")
workflow_execution_id = self.actx.get("workflow_execution_id")
# workflow_name = self.actx.get("workflow_name")
sid = self.psid or workflow_execution_id
# sesCreate return a session ID.
res = {
"content": ses.sesCreate(sid, workflow_execution_id)
}
if self.cmd == "rollback":
res = ses.sesRollback(self.sid)
klog.d(varfmt(res))
return res
def getQuestion(achor):
href = achor.get("href")
dic = db.aoshu.find_one({"q.Url": href})
if dic and dic.get("Content"):
klog.d("Already saved: ", href)
return
dic = DotDict()
qUrl = href
aUrl = ""
Q = []
A = []
klog.d("Geting %s ..." % href)
r = requests.get(href)
content = r.text.encode('ISO-8859-1').decode("utf-8")
soup = BeautifulSoup(content, "html.parser")
title = soup.title.text
mainContent = soup.find("div", {"class": "mainContent"})
pList = mainContent.find_all("p")
for p in pList:
if p.text.find("导语：") >= 0:
continue
if p.text.find("点击下一页") >= 0:
break
t = p.text.strip()
if t:
Q.append(t)
# GetAnswer
aUrl = href[:-5] + "_2.html"
aList = mainContent.find_all("a")
for a in aList:
if a["href"] == aUrl:
if "下一页" in a.text:
A = getAnswer(aUrl)
dic.a.Url = aUrl
dic.q.Url = qUrl
dic.q.Content = Q
dic.a.Content = A
dic.title = title
if aUrl and qUrl and Q and A and title:
db.aoshu.replace_one({"_id": dic.title}, dic.todic(), True)
print(varfmt(dic, title, True))
else:
db.badurl.insert_one({"qUrl": qUrl, "aUrl": aUrl})
def sesRollback(sid):
# Call all the rollback of saved action
klog.d("SID: ", sid)
def gen_rollback_url(url, sid):
urlp = urlparse.urlparse(url)
if urlp.query:
return "%s&rollback=true&sid=%s" % (url, sid)
else:
return "%s?rollback=true&sid=%s" % (url, sid)
try:
client = mgoclient.client()
tab = client.yeehaw.session
res = tab.find_one({"_id": sid})
if not res:
klog.e("BAD SID:", sid)
return False
klog.d(varfmt(res))
info = res.get("info")
kfkMessagePush(
etype=kfk_etypes.EXECUTION_ROLLBACK_START, # etype
exId=info.get("wfexid"), # exId
)
for n in res.get("actions"):
try:
# construct the rollback url
url = gen_rollback_url(n.get("url"), sid)
klog.d(url)
method = n.get("method")
body = n.get("body")
headers = n.get("headers")
params = n.get("params")
actx = n.get("actx")
cookies = n.get("cookies")
auth = n.get("auth")
allow_redirects = n.get("allow_redirects")
proxies = n.get("proxies")
verify = n.get("verify")
# Convert body
if isinstance(body, dict):
body = json.dumps(body, ensure_ascii=False)
if body:
body = body.encode("utf-8")
r = requests.request(method,
url,
params=params,
data=body,
headers=headers,
cookies=cookies,
auth=auth,
allow_redirects=allow_redirects,
proxies=proxies,
verify=verify)
klog.d("RB: code: ", r.status_code, ", reason: ", r.reason)
wf_ex_id = actx.get("workflow_execution_id")
workflow_id = actx.get("workflow_name")
task_id = actx.get("task_id")
kfkMessagePush(
kfk_etypes.TASK_ROLLBACK, # etype
None, # atomId
None, # status
workflow_id, # wfId
wf_ex_id, # exId
task_id, # taskId
None, # taskName
n, # Input
{
"status_code": r.status_code,
"reason": r.reason
}, # Output
None # triggered_by
)
except:
klog.e("RB: NG\n", traceback.format_exc())
retval = True
except:
klog.e(traceback.format_exc())
klog.d("push NG")
retval = False
finally:
# Drop this table
klog.d("drop session information for ", sid)
tab.remove({"_id": sid})
klog.d("drop session information for ", sid, " done.")
client = mgoclient.client()
tab = client.yeehaw.sessionDone
tab.insert_one(res)
kfkMessagePush(
etype=kfk_etypes.EXECUTION_ROLLBACK_END, # etype
exId=info.get("wfexid"), # exId
)
return retval
def __init__(self,
action_context,
url,
method="GET",
params=None,
body=None,
atom=None,
session=None,
headers=None,
cookies=None,
auth=None,
timeout=None,
allow_redirects=None,
proxies=None,
verify=None):
self.ts_init_start = time.time()
klog.d("INTO YIHE.SYNC")
actx = action_context
# 1. Strip the session info
klog.d(" URL : " + str(url))
klog.d("METHOD : " + str(method))
klog.d(" BODY : " + str(body))
klog.d(" HDRS : " + str(headers))
klog.d(" PARAM : " + str(params))
klog.d(" ACTX : " + str(varfmt(actx)))
# FIXME: how to process the atomID
self.atom = atom
self.session = session
sesEna = conf.SES_ENA and session
hookEna = conf.HOOK
# klog.d("Session or NOT:", sesEna, " In config:", conf.SES_ENA, " Session Id:", session)
self.skipRun = False
# Get content information
self.action_execution_id = actx.get("action_execution_id")
self.task_id = actx.get("task_id")
self.task_name = actx.get("task_name")
self.workflow_execution_id = actx.get("workflow_execution_id")
self.workflow_name = actx.get("workflow_name")
if hookEna:
# FIXME: Provide a chance to modify the input and output.
pass
'''
exid = actx.get("action_execution_id")
hookInfo = db_get_hook_info(exid)
if hookInfo:
self.skipRun = hookInfo.skipRun
self.input = hookInfo.input
# Only once
db_mark_used(exid)
#
# Overwrite All the parameters and DO NOT overwrite key
# parameters
#
url = _url or url
body = _body or body
'''
# Overwrite the rerun input and/or output
self.rrInput = self.rrOutput = None
self.ts_ses_start = self.ts_ses_end = 0
if sesEna:
klog.d("Process session")
self.ts_ses_start = time.time()
dic = {
"x__sid": session,
"url": url,
"method": method,
"body": body,
"headers": headers,
"params": params,
"actx": actx,
"cookies": cookies,
"auth": auth,
"allow_redirects": allow_redirects,
"proxies": proxies,
"verify": verify,
}
ses.sesPush(session, dic)
self.ts_ses_end = time.time()
self.orgInit(
url,
method,
params,
body,
headers,
cookies,
auth,
timeout,
allow_redirects,
proxies,
verify,
)
klog.d()
self.ts_init_end = time.time()