def execute(self):
"""
执行工作流
:return: 返回执行id
"""
logger.info('开始执行flow {flow}'.format(flow=self.flowId))
url = '{azkaban_url}/executor?ajax=executeFlow&project={project}&flow={flow}' + self.disabled + self.flow_override
url = url.format(
azkaban_url=azkaban_url,
project=self.prj_name,
flow=self.flowId)
# logger.info("执行url：" + url)
flows_resp = requests.get(
url,
cookies=self.cookies_fetcher.get_cookies()
)
rs = str(flows_resp.content, 'utf-8')
if flows_resp.status_code != 200 or 'error' in rs:
logger.error(rs)
raise Exception('执行{flow} 报错'.format(flow=self.flowId))
else:
# logger.info(rs)
exec_id = json.loads(rs)['execid']
logger.info(('开始执行{flow}，execid是{exec_id}'.format(flow=self.flowId, exec_id=exec_id)))
return FlowExecution(exec_id, self.cookies_fetcher)
def upload_zip(self, zip_file):
"""上传zip文件"""
if self.download_zip():
# 备份文件成功
logger.info("备份{0}项目文件成功".format(self.name))
files = {
'file':
(os.path.basename(zip_file), open(zip_file,
'rb'), 'application/zip')
}
upload_data = {
'project': self.name,
'ajax': 'upload',
}
resp = requests.post(
"{azkaban_url}/manager".format(azkaban_url=azkaban_url),
data=upload_data,
cookies=self.cookies_fetcher.get_cookies(),
files=files)
if resp.status_code != 200:
logger.error(str(resp.content, 'utf-8'))
raise Exception('上传ZIP文件失败：{name} '.format(name=zip_file))
logger.info("上传ZIP文件完成：" + self.name)
else:
raise Exception('项目文件{name}：文件备份失败'.format(name=self.name))
def replace_property(file_name, from_regex, to_str, append_on_not_exists=True):
import tempfile
tmpfile = tempfile.TemporaryFile(mode='w+')
if os.path.exists(file_name):
r_open = open(file_name, 'r')
pattern = re.compile(r'' + from_regex)
found = None
for line in r_open:
if pattern.search(line) and not line.strip().startswith('#'):
found = True
line = re.sub(from_regex, to_str, line)
tmpfile.write(line)
if not found and append_on_not_exists:
tmpfile.write('\n' + to_str)
r_open.close()
tmpfile.seek(0)
content = tmpfile.read()
if os.path.exists(file_name):
os.remove(file_name)
w_open = open(file_name, 'w')
w_open.write(content)
w_open.close()
tmpfile.close()
else:
logger.error("文件 %s not found" % file_name)
def schedule(self, cron=None):
"""
安排执行计划
"""
flow_cron = get_project_info(self.prj_name, "cron." + str(self.flowId).strip())
if flow_cron:
cron = flow_cron # prj_cron.get(self.prj_name + self.flowId, None)
if check_cron(cron):
data = {
'session.id': self.cookies_fetcher.get_session_id(),
'ajax': u'scheduleCronFlow',
'projectName': self.prj_name,
'flow': self.flowId,
'failureAction': 'finishPossible', # finishCurrent, cancelImmediatel
'cronExpression': cron
}
response = requests.post(
azkaban_url + '/schedule',
data=data
)
rs = str(response.content, 'utf-8')
if response.status_code != 200 or 'error' in rs:
logger.info("秒 分 时 日 月 周 data: \n%s", data)
logger.error(rs)
logger.error("{0}设置执行计划失败".format(self.prj_name))
return False
else:
logger.info("{0} flow:{1}设置执行计划成功".format(self.prj_name, self.flowId))
return True
else:
# logger.error("{0}设置执行计划失败,请设置正确的cron时间格式.ERROR for:{1}".format(self.prj_name, cron))
return False
def rplc_cmd_with_prop(cmd, prj_nm, global_prop=None):
"""替换命令里面的参数用来测试"""
if global_prop is None:
global_prop = get_global_prop(prj_nm)
cmd = cmd.replace("$", "")
match = re.findall(r"{.*?}", cmd) # r"\{.*?\}"
for i in match:
if "." in i:
cmd = cmd.replace(i, i.replace(".", "_"))
try:
cmd = cmd.format(**global_prop)
if check_file_exists: # 检查文件是否存在
tp = cmd.split(" ")
for i in tp[0:2]: # 仅检查前两个
if "." in i:
file_nm, file_type = os.path.splitext(i)
if file_type in ['.sh', '.py', '.jar']:
if i.startswith("/") or i.startswith("\\"):
file_nm = i
else:
file_nm = os.path.join(get_prj_path(prj_nm, file_type="dir", search_path="conf"), i)
if not os.path.exists(file_nm):
if is_prod:
logger.error("项目文件不存在：" + file_nm)
return None
else:
logger.warning("测试环境，项目文件不存在：" + file_nm)
return cmd
except Exception as e:
errors = str(e)
if 'azkaban_' in errors:
errors = errors.replace("_", ".")
logger.warning("使用参数在全局变量中不存在：" + errors)
return cmd
def exec_shell(shell, logs_print=True):
logger.info("执行shell:" + shell)
try:
proc = subprocess.Popen(shell,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) # subprocess.STDOUT
pre_line = ''
error_line = ''
while proc.poll() is None:
line = proc.stdout.readline().strip().decode('utf-8')
if line and len(line) > 2:
pre_line = line
if 'error' in line.lower():
error_line = line
if logs_print:
logger.info(line)
# outs, errs = proc.communicate(timeout=15)
# res是一个对象，需要读取res对象的stderr\strout\stdin的属性，才可获取值
# 如果：err有值，即表示命令执行报错，即：stdout就为空
if proc.returncode == 0:
logger.info('shell 执行成功')
return True
else:
raise Exception(error_line + '\n' + pre_line)
except Exception as e:
logger.error('shell执行失败：' + str(e))
raise Exception("shell执行失败:" + str(e))
def eval_str(strs):
if strs:
try:
rs = eval(strs)
return rs
except Exception as e:
logger.error("参数格式不对 -f/-i/-d \"['a','b']\" -p \"{'s':1}\" :" + str(e))
sys.exit(0)
else:
return None
def fetch_flow(self):
"""获取所有工作流"""
flows_resp = requests.get(
'{azkaban_url}/manager?ajax=fetchprojectflows&project={project}'.format(azkaban_url=azkaban_url, project=self.name),
cookies=self.cookies_fetcher.get_cookies())
if flows_resp.status_code != 200:
logger.error(str(flows_resp.content, 'utf-8'))
raise Exception('Error happened when fetch flow from {project} in azkaban'.format(project=self.name))
flows = json.loads(str(flows_resp.content, 'utf-8'))['flows']
tp = []
for flow in flows:
tp.append(flow['flowId'])
# yield Flow(self.name, flow['flowId'], cookies_fetcher=self.cookies_fetcher)
return tp
def __init__(self, user=login_user, pwd=login_pwd):
self.login_data = {
'action': 'login',
'username': user,
'password': pwd
}
try:
resp = requests.post(
"{azkaban_url}".format(azkaban_url=azkaban_url),
data=self.login_data)
self.cookies = resp.cookies
self.session_id = resp.json()['session.id']
logger.debug("azkaban登录成功")
except Exception as e:
logger.error("登录错误：" + str(e))
def schedule_flows(self, cron=None, flows=None):
"""将所有的工作流列入执行计划"""
all_flows = self.fetch_flow()
if cron is None:
# try:
cron = get_project_info(prj_nm=self.name, key_nm="cron." + self.name) # prj_cron.get(self.name, None)
if cron is None and get_project_info(prj_nm=self.name, key_nm="cron") is None:
logger.error(self.name + "的 system.properties中没有配置定时器cron请配置")
return
if flows is None:
flows = all_flows
for f in all_flows:
if f in flows:
logger.info("设置项目{0}的工作流{1}执行计划".format(self.name, f))
flow = Flow(self.name, f, self.cookies_fetcher)
flow.schedule(cron)
def extract_json(response):
"""
解析 JSON 来自 response.
:param response: Request response object.
"""
try:
json = response.json()
except ValueError as err: # this should never happen
logger.error('没有json格式体 :\n%s', str(response.text, 'utf-8'))
raise err
else:
if 'error' in json:
raise json['error']
elif json.get('status') == 'error':
raise json['message']
else:
return json
def check_cron(cron):
"""
检查cron时间格式是否合规
:param cron:
:return:
"""
if cron and " " in cron:
try:
pt = Cron(cron)
logger.info("cron格式校验通过,下次执行时间: " + str(pt.get_next()))
return True
except Exception as e:
logger.error(str(e))
logger.error("cron不能为空或者格式不对,你输入的是{0}".format(cron))
return False
else:
logger.warning("cron不能为空或者格式不对,你输入的是{0}".format(cron))
return False
def unscheduled(self):
"""
取消执行计划
"""
schd_id = self.fetch_schedule()
if schd_id:
data = {
u'session.id': self.cookies_fetcher.get_session_id(),
u'action': u'removeSched',
u'scheduleId': schd_id
}
response = requests.post(azkaban_url + '/schedule', data=data)
if response.status_code != 200:
logger.info("Request data: \n%s", data)
logger.error(str(response.content, 'utf-8'))
raise Exception("{0} 取消执行计划失败".format(self.prj_name))
else:
logger.info("{0} flow:{1} 执行计划id={2}已经被取消".format(
self.prj_name, self.flowId, schd_id))
return True
def get_project_info(prj_nm, key_nm):
"""
获取项目的参数配置,例如dw.properties
:param prj_nm
:param key_nm 如a.b.c a
:return
"""
# prj_path = os.path.join(conf_path, prj_nm)
# prop_file_sys = os.path.join(prj_path, "system.properties")
prop_file_prj = get_prj_path(prj_nm, file_type="properties", search_path="conf") # os.path.join(prj_path, prj_nm + ".properties")
prop_file = None
if os.path.exists(prop_file_prj):
prop_file = prop_file_prj
if prop_file:
prop = Properties(prop_file)
prop.get_properties() # 获取数值
return prop.get_value_by_key(key_nm)
else:
logger.error(prop_file_prj + "配置文件不存在，请加以配置")
return None
def copy_local_param(prj_nm):
"""读取项目下的配置文件内容，并输出到正式项目system.properties"""
rd_file = get_prj_path(prj_nm, file_type="properties", search_path="conf")
# os.path.join(os.path.join(conf_path, prj_nm), prj_nm + ".properties")
wt_file = get_prj_path(prj_nm, file_type="properties", search_path="temp")
# os.path.join(os.path.join(temp_path, prj_nm), "system.properties")
if not os.path.exists(rd_file):
logger.error(rd_file + "文件不存在，请创建")
return
if not os.path.exists(wt_file):
logger.error(wt_file + "参数文件没有生成")
return
with open(wt_file, 'a') as w:
with open(rd_file, 'r') as r:
line = r.readline()
while line:
if "=" in line.strip():
tp = line.strip().split("=")[0].strip()
if tp in job_config_keys or 'cron' in tp:
# 特殊参数 和job参数不输入到systemp.properties
pass
else:
w.write(line)
line = r.readline()
def active_executor(hosts=None, port=12321):
"""激活各节点你的Executor"""
unactive_host_list = get_executor(active=0)
if len(unactive_host_list) < 1:
logger.info("所有节点都已经激活")
return
if hosts:
if hosts not in unactive_host_list:
logger.info("{0}节点没有部署成功，请确认是否部署成功和hostname".format(hosts))
return
url = "http://{0}:{1}/executor?action=activate".format(hosts, port)
try:
rs = requests.get(url)
if str(rs.content, 'utf-8') == "{\"status\":\"success\"}":
logger.info("Executor : {0} 激活成功".format(hosts))
else:
logger.error("激活Executor失败,请确认Executor正确启动")
except Exception as e:
logger.error("激活Executor失败,请确认正确的ip和port" + str(e))
else:
rs = get_executor(active=0, rstype="port")
for i in rs.keys():
active_executor(i, rs[i])
def crt_job_file(prj_nm):
"""
创建job文件
:param prj_nm:
:return:
"""
prj_conf_path = os.path.join(conf_path, prj_nm)
filepath = os.path.join(prj_conf_path, prj_nm + '.csv')
if os.path.exists(filepath):
df = pd.read_csv(filepath)
prj_path = os.path.join(temp_path, prj_nm)
df['command'] = df['command'].fillna(
"echo 'Exec ${azkaban.job.id} of ${azkaban.flow.flowid} with batch_id:${batch_id} etl_dt:${etl_dt} batch_dt:${batch_dt}'"
)
if os.path.exists(prj_path):
import shutil
shutil.rmtree(prj_path) # 清空项目对应的临时目录，一般是temp目录下
os.makedirs(prj_path)
df = df.fillna('')
df['job_nm'] = df['job_nm'].apply(lambda x: x.strip())
jobs = set(df['job_nm']) # 需要配置的job set
df['dependencies'] = df['dependencies'].apply(
lambda x: x.strip().replace("，", ",").replace(" ", ""))
df['dependencies'].apply(
lambda x: check_depend_if_in_flow(x, jobs)) # 检查依赖job是否在配置中
for i in df.to_dict(orient='records'):
job_path = os.path.join(prj_path, i['job_nm'] + '.job')
with open(job_path, 'w') as job:
job.write("type = command\n")
if len(i['dependencies']) > 1:
job.write("dependencies = " +
i['dependencies'].strip().replace(
"，", ",").replace(" ", "") + "\n")
job.write("retries = {0}\n".format(
get_project_info(prj_nm, "retries") or 3))
job.write("retry.backoff = {0}\n".format(
get_project_info(prj_nm, "retry.backoff")
or 60000)) # 重试的间隔（毫秒）
if get_project_info(prj_nm, "failure.emails"):
# 失败邮件通知 如果项目配置文件有配置
job.write("failure.emails = {0}\n".format(
get_project_info(prj_nm, "failure.emails")))
if get_project_info(prj_nm, "success.emails"):
# 失败邮件通知 如果项目配置文件有配置
job.write("success.emails = {0}\n".format(
get_project_info(prj_nm, "success.emails")))
if get_project_info(prj_nm, "working.dir"):
# 失败邮件通知 如果项目配置文件有配置
job.write("working.dir = {0}\n".format(
get_project_info(prj_nm, "working.dir")))
# command = "command = {0} {1} {2} {3}".format(i['command'].strip(), i['arg1'], i['arg2'], i['arg3'])
command = "command = {0}".format(i['command'].strip())
check_cmd = rplc_cmd_with_prop(i['command'].strip(), prj_nm)
if check_cmd:
job.write(command.strip())
else:
logger.error("{0}项目job文件生中断，命令解析不通过".format(prj_nm))
raise Exception("{0}项目job文件生中断，命令解析不通过".format(prj_nm))
# 生成end_flow
depend_job_set = set() # 依赖的job set 集
def add_depend_job_set(strs):
"""计算被依赖的job"""
if strs:
tp = strs.split(",")
depend_job_set.update(tp) # 更新 job set
df[df['dependencies'].notna()]['dependencies'].apply(
add_depend_job_set)
not_depended_jobs = ','.join(jobs - depend_job_set) # 算出不被依赖的job
job_path = os.path.join(prj_path, prj_nm + "_end_flow.job")
with open(job_path, 'w') as job: # 生成end_flow_prj_nm.job 文件
job.write("type = command\n")
job.write("dependencies = " + not_depended_jobs + "\n")
job.write("command = echo 'all flow end' \n")
# end_flow job生成完成
crt_sys_prop(prj_nm) # 生成全局系统参数文件 system.properies
copy_local_param(prj_nm) # 把项目参数（例如dw.properies）输出到全局系统参数文件
# copy_dir(prj_conf_path, os.path.join(prj_path, 'scripts'), ignore_file_type=['job', 'flow', 'project'])
# copy_dir(prj_conf_path, prj_path, only_file_type=['job', 'flow', 'project'])
copy_dir(prj_conf_path, prj_path, ignore_file_type=['properties'])
zip_path = prj_path + '.zip'
zip_dir(prj_path, zip_path) # 将目标目录压缩成zip文件
# logger.info("{0}项目压缩完成，文件路径是：{1}".format(prj_nm, zip_path))
return zip_path # 返回zip文件路径
else:
logger.error("文件不存在:" + filepath)
return None
