def collectFiles(self, ftpDir):
try:
self.ftp.cwd(ftpDir)
except:
print("NG: CWD:", ftpDir)
return
listing = []
self.ftp.retrlines("LIST", listing.append)
for l in listing:
# Get name and size
words = l.split(None, 8)
isdir = words[0][0] == 'd'
size = int(words[4])
name = words[-1]
ftpPath = ftpDir + "/" + name
if isdir:
self.collectFiles(ftpPath)
else:
localPath = os.path.join(self.baseDir, ftpPath[1:])
if self.okExt(localPath, theExts) and self.okSize(size, 100):
f = File(ftpPath, localPath, size)
self.foundFiles.append(f)
# If already done
if self.inLocal(localPath, size) or self.inDatabase(localPath, size):
klog.d("SKIP: %s : %s" % (commaNum(size), localPath))
self.skipFiles.append(f)
continue
self.ingFiles.append(f)
klog.d("WGET: %16s : %s" % (commaNum(size), localPath))
def sesPush(sid, inf):
# Add action node
"""
p.update({"_id":sid},{"$push":{"actions":"Test"}})
p.update({"_id":sid},{"$addToSet": {"tags":{"$each":["Python","Each"]}}})
"""
try:
# klog.d("SID: ", sid, ", INF: ", varfmt(inf))
client = mgoclient.client()
tab = client.yeehaw.session
res = tab.find_one({"_id": sid})
if not res:
klog.e("push NG: sid %s not exist" % sid)
return False
inf["createat"] = time.time()
tab.update({"_id": sid}, {"$push": {"actions": inf}})
klog.d("push OK")
return True
except:
klog.e(traceback.format_exc())
klog.d("push NG")
return False
def _schedule_actions(self):
# Regular task schedules just one action.
input_dict = self._get_action_input()
klog.d("action inputs:", input_dict)
# If task action is rerun, replace input data
if hasattr(self, '_task_action'):
if self._task_action == 'rerun':
input_dict = self._task_input
elif self._task_action == 'pass':
# input_dict = {"output":str(self._task_output)}
input_dict = {"output": self._task_output}
# Add Kafka log tracel
atom_id = None
if input_dict.has_key('atom'):
atom_id = input_dict['atom']
kfk_trace.log(kfk_etypes.TASK_RUNNING, atom_id, states.RUNNING,
self.wf_ex.workflow_id, self.wf_ex.id,
self.task_ex.id, self.task_ex.name,
input_dict, None, self.triggered_by)
target = self._get_target(input_dict)
action = self._build_action()
action.validate_input(input_dict)
action.schedule(
input_dict,
target,
safe_rerun=self.task_spec.get_safe_rerun()
)
def get_app_url(app_name):
"""
:param app_name: string, name of app when register
:return:
"""
app_url = None
if conf.EUREKA_ENABLE:
url = conf.EUREKA_URL + "/apps/" + app_name
try:
resp = requests.get(url=url, headers={"Accept": "application/json"}, timeout=0.5)
klog.d("Query app <", app_name, "> url:", resp.status_code, resp.content)
if resp.ok:
r = resp.json()
app_url = _get_app_url(r)
except Exception as e:
klog.d("Eureka Error: ", e)
if not app_url:
if app_name==conf.APP_NAME_API:
return conf.YIHE_API_HOST
elif app_name==conf.APP_NAME_CALLBACK:
return conf.YIHE_ASYNC_CALLBACK_HOST
return app_url
def timefn(*args, **kwargs):
if conf.DB_TRACE:
beg = time.time() * 1000
result = fn(*args, **kwargs)
end = time.time() * 1000
klog.d("%s : %f ms." % (fn.func_name.rjust(36), (end - beg)))
else:
result = fn(*args, **kwargs)
return result
def mque_put(self, msg, key=None, wakeup=True, left=False):
klog.d(msg, ":", key)
self.lock.acquire()
try:
if left:
self.mque.appendleft((msg, key))
else:
self.mque.append((msg, key))
if wakeup:
self.wakeup()
except:
pass
finally:
self.lock.release()
def __init__(self,
action_context,
url,
method="GET",
params=None,
body=None,
atom=None,
session=None,
headers=None,
cookies=None,
auth=None,
timeout=None,
allow_redirects=None,
proxies=None,
verify=None):
actx = action_context
ck_url = actx.get('callback_url')
if conf.EUREKA_ENABLE:
host = eureka.get_app_url(conf.APP_NAME_CALLBACK)
ck_url = host + ck_url
klog.d("Eureka enable, CK_URL:", ck_url)
headers = headers or {}
headers.update({
'Workflow-Name': actx.get('workflow_name'),
'Workflow-Execution-Id': actx.get('workflow_execution_id'),
'Task-Id': actx.get('task_id'),
'Action-Execution-Id': actx.get('action_execution_id'),
'Callback-URL': ck_url,
})
super(Async, self).__init__(
action_context,
url,
method,
params,
body,
atom,
session,
headers,
cookies,
auth,
timeout,
allow_redirects,
proxies,
verify,
)
def act(self):
while True:
ret = self.mque_pop_first()
if not ret:
break
msg, key = ret
try:
client = self.mgoclient.client().yihe
dic = {"msg": msg, "key": key}
client.evtlog.insert_one(dic)
klog.d("Push one message to mongodb...")
except:
klog.e(traceback.format_exc())
self.mque_put(msg=msg, key=key, left=True)
return 60
def __init__(self, action_context, cmd, psid=None, sid=None):
actx = action_context
klog.d(" ACTX : " + str(varfmt(actx)))
klog.d(" CMD : " + str(cmd))
klog.d(" PSID : " + str(psid))
klog.d(" SID : " + str(sid))
self.cmd = cmd
self.actx = actx
self.psid = psid
self.sid = sid
def kfkMessagePush(etype=None,
atomId=None,
status=None,
wfId=None,
exId=None,
taskId=None,
taskName=None,
input=None,
output=None,
triggered_by=None):
etype = etype or "Unknown"
wfId = wfId or "x01"
exId = exId or "x02"
taskId = taskId or ""
taskName = taskName or ""
atomId = atomId or ""
status = status or "unknown"
input = input or {}
output = output or {}
triggered_by = triggered_by or []
try:
message = kfk_trace.KfkMessage(etype, wfId, exId, taskId, taskName,
atomId, status, input, output,
triggered_by)
msg = message.to_str()
key = None
klog.d(msg, ":", key)
client = mgoclient.client().yihe
dic = {"msg": msg, "key": key}
client.evtlog.insert_one(dic)
except:
klog.e(
"Kafka Message build fail: ",
"etype=%s,wfId=%s,exId=%s,taskId=%s,taskName=%s" %
(etype, wfId, exId, taskId, taskName))
klog.e(traceback.format_exc())
pass
def inLocal(self, localPath, size):
if not os.path.exists(localPath):
klog.d("TARGET: NOT EXISTS:", localPath)
return False
try:
localsize = os.path.getsize(localPath)
if size == localsize:
klog.d("TARGET: SAME SIZE:(%s): %s" % (commaNum(size), localPath))
return True
else:
klog.d("TARGET: DIFF SIZE:(%s/%s): %s" % (commaNum(size), commaNum(localsize), localPath))
return False
except:
traceback.print_exc()
return False
def sesCreate(psid, wfexid):
# Create a new session
klog.d("Parent Session ID: ", psid)
try:
client = mgoclient.client()
tab = client.yeehaw.session
res = tab.find_one({"_id": psid})
if res:
klog.d("PSID %s found" % str(psid))
return psid
info = {
"wfexid": wfexid,
}
res = tab.insert_one({"_id": psid, "info": info})
klog.d("SID %s added" % res.inserted_id)
return str(res.inserted_id)
except:
klog.e(traceback.format_exc())
return "OK"
def run(self, context):
klog.d("CMD: ", self.cmd)
klog.d(varfmt(context))
if self.cmd == "create":
# Get content information
# action_execution_id = self.actx.get("action_execution_id")
# task_id = self.actx.get("task_id")
# task_name = self.actx.get("task_name")
workflow_execution_id = self.actx.get("workflow_execution_id")
# workflow_name = self.actx.get("workflow_name")
sid = self.psid or workflow_execution_id
# sesCreate return a session ID.
res = {
"content": ses.sesCreate(sid, workflow_execution_id)
}
if self.cmd == "rollback":
res = ses.sesRollback(self.sid)
klog.d(varfmt(res))
return res
import sys
import time
import traceback
from random import randint
from ftplib import FTP
import multiprocessing
import subprocess
from mie.xlogger.klog import klog
from mie.deferdo import DeferDo
klog.to_file("/tmp/aaa.txt")
klog.d("START ...")
klog.d("START ...")
klog.d("START ...")
klog.d("START ...")
klog.d("START ...")
klog.d("START ...")
klog.d("START ...")
klog.d("START ...")
klog.d("START ...")
klog.d("START ...")
klog.d("START ...")
okDirs = (
"/MIUI/sound_recorder",
"/DCIM",
"/Tencent/MicroMsg/weixin",
def sesRollback(sid):
# Call all the rollback of saved action
klog.d("SID: ", sid)
def gen_rollback_url(url, sid):
urlp = urlparse.urlparse(url)
if urlp.query:
return "%s&rollback=true&sid=%s" % (url, sid)
else:
return "%s?rollback=true&sid=%s" % (url, sid)
try:
client = mgoclient.client()
tab = client.yeehaw.session
res = tab.find_one({"_id": sid})
if not res:
klog.e("BAD SID:", sid)
return False
klog.d(varfmt(res))
info = res.get("info")
kfkMessagePush(
etype=kfk_etypes.EXECUTION_ROLLBACK_START, # etype
exId=info.get("wfexid"), # exId
)
for n in res.get("actions"):
try:
# construct the rollback url
url = gen_rollback_url(n.get("url"), sid)
klog.d(url)
method = n.get("method")
body = n.get("body")
headers = n.get("headers")
params = n.get("params")
actx = n.get("actx")
cookies = n.get("cookies")
auth = n.get("auth")
allow_redirects = n.get("allow_redirects")
proxies = n.get("proxies")
verify = n.get("verify")
# Convert body
if isinstance(body, dict):
body = json.dumps(body, ensure_ascii=False)
if body:
body = body.encode("utf-8")
r = requests.request(method,
url,
params=params,
data=body,
headers=headers,
cookies=cookies,
auth=auth,
allow_redirects=allow_redirects,
proxies=proxies,
verify=verify)
klog.d("RB: code: ", r.status_code, ", reason: ", r.reason)
wf_ex_id = actx.get("workflow_execution_id")
workflow_id = actx.get("workflow_name")
task_id = actx.get("task_id")
kfkMessagePush(
kfk_etypes.TASK_ROLLBACK, # etype
None, # atomId
None, # status
workflow_id, # wfId
wf_ex_id, # exId
task_id, # taskId
None, # taskName
n, # Input
{
"status_code": r.status_code,
"reason": r.reason
}, # Output
None # triggered_by
)
except:
klog.e("RB: NG\n", traceback.format_exc())
retval = True
except:
klog.e(traceback.format_exc())
klog.d("push NG")
retval = False
finally:
# Drop this table
klog.d("drop session information for ", sid)
tab.remove({"_id": sid})
klog.d("drop session information for ", sid, " done.")
client = mgoclient.client()
tab = client.yeehaw.sessionDone
tab.insert_one(res)
kfkMessagePush(
etype=kfk_etypes.EXECUTION_ROLLBACK_END, # etype
exId=info.get("wfexid"), # exId
)
return retval
def run(self, context):
self.ts_run_start = time.time()
if not self.skipRun:
# TODO: replace input with self.rrInput
# if self.rrInput:
# pass
klog.d(varfmt(self))
# klog.d(varfmt(context))
klog.d(">>> Run action")
res = self.orgRun(context)
klog.d("<<< Run action")
klog.d(varfmt(res))
try:
_input = todict(self)
except:
_input = None
try:
_output = todict(res)
except:
_output = None
kfk_trace.log(kfk_etypes.TASK_ACTION, self.atom, "RUNNING",
self.workflow_name, self.workflow_execution_id,
self.task_id, self.task_name,
_input, _output, None)
else:
klog.d("YIHE.Sync.run skipped")
# TODO: replace output with self.rrOutput
if self.rrOutput:
pass
self.ts_run_end = time.time()
klog.d("TIME: Init total: ", self.ts_init_end - self.ts_init_start)
klog.d("TIME: Run total: ", self.ts_run_end - self.ts_run_start)
klog.d("TIME: Action Total: ", self.ts_run_end - self.ts_init_start)
if self.ts_ses_end:
klog.d("TIME: Session Total: ", self.ts_ses_end - self.ts_ses_start)
return res
def __init__(self,
action_context,
url,
method="GET",
params=None,
body=None,
atom=None,
session=None,
headers=None,
cookies=None,
auth=None,
timeout=None,
allow_redirects=None,
proxies=None,
verify=None):
self.ts_init_start = time.time()
klog.d("INTO YIHE.SYNC")
actx = action_context
# 1. Strip the session info
klog.d(" URL : " + str(url))
klog.d("METHOD : " + str(method))
klog.d(" BODY : " + str(body))
klog.d(" HDRS : " + str(headers))
klog.d(" PARAM : " + str(params))
klog.d(" ACTX : " + str(varfmt(actx)))
# FIXME: how to process the atomID
self.atom = atom
self.session = session
sesEna = conf.SES_ENA and session
hookEna = conf.HOOK
# klog.d("Session or NOT:", sesEna, " In config:", conf.SES_ENA, " Session Id:", session)
self.skipRun = False
# Get content information
self.action_execution_id = actx.get("action_execution_id")
self.task_id = actx.get("task_id")
self.task_name = actx.get("task_name")
self.workflow_execution_id = actx.get("workflow_execution_id")
self.workflow_name = actx.get("workflow_name")
if hookEna:
# FIXME: Provide a chance to modify the input and output.
pass
'''
exid = actx.get("action_execution_id")
hookInfo = db_get_hook_info(exid)
if hookInfo:
self.skipRun = hookInfo.skipRun
self.input = hookInfo.input
# Only once
db_mark_used(exid)
#
# Overwrite All the parameters and DO NOT overwrite key
# parameters
#
url = _url or url
body = _body or body
'''
# Overwrite the rerun input and/or output
self.rrInput = self.rrOutput = None
self.ts_ses_start = self.ts_ses_end = 0
if sesEna:
klog.d("Process session")
self.ts_ses_start = time.time()
dic = {
"x__sid": session,
"url": url,
"method": method,
"body": body,
"headers": headers,
"params": params,
"actx": actx,
"cookies": cookies,
"auth": auth,
"allow_redirects": allow_redirects,
"proxies": proxies,
"verify": verify,
}
ses.sesPush(session, dic)
self.ts_ses_end = time.time()
self.orgInit(
url,
method,
params,
body,
headers,
cookies,
auth,
timeout,
allow_redirects,
proxies,
verify,
)
klog.d()
self.ts_init_end = time.time()