def vars_map(self, key, value, init_day, format=None):
init_date = DateUtil.parse_date(init_day, None)
if key == 'today':
if value is None:
return DateUtil.get_now_fmt(format, init_date)
else:
return value
elif key == 'yesterday':
if value is None:
return DateUtil.get_yesterday_fmt(format, init_date)
else:
return value
elif key == 'intervalday':
if value is None:
raise Exception("interval day is none")
return DateUtil.get_interval_day_fmt(int(value), format, init_date)
elif key == 'lastMonth':
if value is None:
return DateUtil.get_last_month(init_date)
else:
return value
elif key == 'currentMonth':
if value is None:
return DateUtil.get_current_month(init_date)
else:
return value
elif key == 'yesterdayMonth':
if value is None:
return DateUtil.get_yesterday_month(init_date)
else:
return value
else:
return value
def run():
date_str = DateUtil.get_now_fmt("%Y%m%d")
query_date_str = date_str + "0" * 10
zeus_connection = get_zeus_connection()
query_sql = """select status,count(id) as action_count from zeus_action where id >= %s group by status """
cursor = zeus_connection.cursor(MySQLdb.cursors.DictCursor)
cursor.execute(query_sql, (query_date_str, ))
rows = cursor.fetchall()
null_count = 0
success_count = 0
running_count = 0
failed_count = 0
other_count = 0
for row in rows:
status = row["status"]
if status is None:
null_count = row["action_count"]
elif status == "failed":
failed_count = row["action_count"]
elif status == "running":
running_count = row["action_count"]
elif status == "success":
success_count = row["action_count"]
else:
print("other:" + str(status))
other_count = row["action_count"]
total_count = null_count + success_count + running_count + failed_count + other_count
msg = [
"总的任务数:" + str(total_count), "未运行任务数:" + str(null_count),
"运行中任务数:" + str(running_count), "运行完成任务数:" + str(success_count),
"运行失败任务数:" + str(failed_count)
]
content = date_str + " 运行日志信息:\n" + str(",\n".join(msg)) + "\n"
query_user_sql = """select phone from zeus_user where is_effective = 1 and uid !='admin' """
cursor.execute(query_user_sql, ())
rows = cursor.fetchall()
phones = set()
for row in rows:
phones.add(row["phone"])
response = smsUtil.send(",".join(phones), content)
print(response)
def run(self):
today = DateUtil.get_now_fmt(None)
msg = []
connection = self.dbUtil.get_connection()
cursor = connection.cursor(MySQLdb.cursors.DictCursor)
self.run_count(cursor, today, msg)
self.run_check(cursor, today, msg)
connection.close()
main_phones = self.dboption.get_main_man_by_role("admin")
phones = set()
for main_phone in main_phones:
phones.add(main_phone['user_phone'])
if not phones or len(phones) == 0:
print("没有配置短信发送phone")
return
content = today + " 运行日志信息:\n" + str(",\n".join(msg))
response = self.smsUtil.send(",".join(phones), content)
print(response)
def get_dependency(cursor, job_name, dep_jobs):
dep_sql = "select job_name,dependency_job from t_etl_job_dependency where job_name = %s"
cursor.execute(dep_sql, (job_name,))
deps = cursor.fetchall()
for dep_job in deps:
dep_jobs.add(dep_job["dependency_job"])
get_dependency(cursor, dep_job["dependency_job"], dep_jobs)
return dep_jobs
if __name__ == '__main__':
dbUtil = DBUtil()
connection = dbUtil.get_connection()
cursor = connection.cursor(MySQLdb.cursors.DictCursor)
today = DateUtil.get_now_fmt()
job_sql = "select job_name,last_start_time,last_end_time from t_etl_job where 1=1 "
cursor.execute(job_sql + " and last_run_date=%s", (today,))
jobs = cursor.fetchall()
count = 0
failed = 0
error = 0
for job in jobs:
job_name = job["job_name"]
job_start_time = datetime.datetime.strptime(job["last_start_time"], "%Y-%m-%d %H:%M:%S")
dep_jobs = set()
get_dependency(cursor, job_name, dep_jobs)
for dep_job in dep_jobs:
cursor.execute(job_sql + " and job_name = %s", (dep_job,))
def export_command(self, python_path, project_path, command_key,
command_value, init_day):
mysql2hive = project_path + '/export/mysql2hive.py'
mongo2hive = project_path + '/export/mongo2hive.py'
hive2mysql = project_path + '/export/hive2mysql.py'
hive2excel = project_path + '/export/hive2excel.py'
odps2hive = project_path + '/export/odps2hive.py'
command_list = []
command_list += python_path
if command_key == 'mysql2hive':
command_list.append(mysql2hive)
command_list.append("--from")
# mysql 分表处理
mysql_db = self.replace_mysql_db(command_value['mysql_db'],
init_day)
command_list.append(mysql_db)
command_list.append("--to")
command_list.append(command_value['hive_db'])
if command_value.has_key(
"include_columns") and command_value['include_columns']:
command_list.append("--columns")
command_list.append(command_value['include_columns'])
if command_value.has_key(
"exclude_columns") and command_value['exclude_columns']:
command_list.append("--exclude-columns")
command_list.append(command_value['exclude_columns'])
vars = {}
if command_value.has_key("vars") and command_value["vars"]:
vars = command_value["vars"]
if command_value.has_key(
"partition") and command_value['partition']:
command_list.append("--partition")
partition_value = command_value['partition']
partition_value = self.replace_sql_param(
partition_value, vars, init_day)
command_list.append(partition_value)
if command_value.has_key("where") and command_value['where']:
command_list.append("--where")
partition_value = command_value['where']
partition_value = self.replace_sql_param(
partition_value, vars, init_day)
command_list.append(partition_value)
if command_value.has_key(
"query_sql") and command_value['query_sql']:
command_list.append("--query-sql")
command_list.append(command_value['query_sql'])
return command_list
if command_key == 'mongo2hive':
command_list.append(mongo2hive)
command_list.append("--file")
command_list.append(command_value["yaml_file"])
command_list.append("--from")
command_list.append(command_value["mongo_db"])
command_list.append("--to")
command_list.append(command_value["hive_db"])
command_list.append("--init")
if init_day is None:
init_day = DateUtil.get_now_fmt()
command_list.append(init_day)
vars = {}
if command_value.has_key("vars") and command_value["vars"]:
vars = command_value["vars"]
if command_value.has_key(
'partition') and command_value['partition']:
command_list.append("--partition")
partition_value = command_value['partition'].strip()
partition_value = self.replace_sql_param(
partition_value, vars, init_day)
command_list.append(partition_value)
return command_list
if command_key == 'hive2mysql':
command_list.append(hive2mysql)
vars = {}
if command_value.has_key("vars") and command_value["vars"]:
vars = command_value["vars"]
if command_value.has_key(
"delete_sql") and command_value["delete_sql"]:
command_list.append("--sql")
sql = self.replace_sql_param(command_value["delete_sql"], vars,
init_day)
command_list.append(sql)
if command_value.has_key("query") and command_value["query"]:
command_list.append("--query")
hql = self.replace_sql_param(command_value["query"], vars,
init_day)
command_list.append(hql)
command_list.append("--hive")
command_list.append(command_value['hive_db'])
command_list.append("--to")
command_list.append(command_value['mysql_db'])
command_list.append("--columns")
command_list.append(command_value['mysql_columns'])
return command_list
if command_key == 'hive2excel':
command_list.append(hive2excel)
vars = {}
if command_value.has_key("vars") and command_value["vars"]:
vars = command_value["vars"]
command_list.append("--name")
command_list.append(command_value['excel_name'])
command_list.append("--subject")
command_list.append(command_value['email_subject'])
command_list.append("--content")
command_list.append(command_value['email_content'])
if command_value.has_key("hive_db") and command_value["hive_db"]:
command_list.append("--table")
command_list.append(command_value['hive_db'])
command_list.append("--receivers")
command_list.append(command_value['email_receivers'])
if command_value.has_key("query") and command_value["query"]:
command_list.append("--query")
hql = self.replace_sql_param(command_value["query"], vars,
init_day)
command_list.append(hql)
return command_list
if command_key == 'odps2hive':
command_list.append(odps2hive)
command_list.append("--from")
command_list.append(command_value['odps_db'])
command_list.append("--to")
command_list.append(command_value['hive_db'])
if command_value.has_key(
"include_columns") and command_value['include_columns']:
command_list.append("--columns")
command_list.append(command_value['include_columns'])
if command_value.has_key(
"exclude_columns") and command_value['exclude_columns']:
command_list.append("--exclude-columns")
command_list.append(command_value['exclude_columns'])
vars = {}
if command_value.has_key("vars") and command_value["vars"]:
vars = command_value["vars"]
if command_value.has_key(
'partition') and command_value['partition']:
command_list.append("--partition")
partition_value = command_value['partition'].strip()
partition_format = None
if command_value.has_key('partition_format') and command_value[
'partition_format']:
partition_format = command_value['partition_format'].strip(
)
partition_value = self.replace_sql_param(
partition_value, vars, init_day, partition_format)
command_list.append(partition_value)
return command_list
reload(sys)
sys.setdefaultencoding('utf-8')
run_yaml = RunYaml()
optParser = run_yaml.option_parser()
options, args = optParser.parse_args(sys.argv[1:])
if options.path is None:
print("require yaml file")
optParser.print_help()
sys.exit(-1)
start = options.start
end = options.end
if start is None:
start = DateUtil.parse_date(DateUtil.get_now_fmt())
else:
start = DateUtil.parse_date(start)
if end is None:
end = DateUtil.parse_date(DateUtil.get_now_fmt())
else:
end = DateUtil.parse_date(end)
run_code = []
for p_day in DateUtil.get_list_day(start, end):
print "运行时设置的日期:", p_day
code = run_yaml.run_command(options.path, p_day)
run_code.append(str(code))
if code != 0:
sys.exit(1)
code_str = ",".join(run_code)