def put(self, id):
obj = ScheduleModel.get_by_id(id)
if obj:
self.parser.add_argument("data", type=str, required=True)
args = self.parser.parse_args()
data = json.loads(args["data"])
try:
obj.update(data)
except Exception as e:
print(e)
return {"msg": "Update error!", "data": ""}, 404
return {"data": obj.to_dict(), "msg": ""}
return {"msg": "Item not found!", "data": ""}, 404
def schedule():
with app.app_context():
users = UserModel.query.all()
ScheduleModel.query.delete()
amizone = amizonebot()
for user in users:
amizone.login(user.username, user.password)
schedule = amizone.getSchedule()
i = 2
while i < len(schedule):
ScheduleModel(
user_id=user.id, course_details=schedule[i], prof_name=schedule[i+1]).save_to_db()
i = i+2
def delete(self, name):
movie = MovieModel.get_movie(name)
logger.info("Validating request data")
if not movie:
return {"error": f"No Movie with the name {name}"}, 404
schedules = ScheduleModel.get_schedule(movie_id=movie.id)
if len(schedules) > 0:
logger.info("Schedule not found")
return {
"error": f"Delete schedule of the movie before deleting it"
}, 400
movie.delete_from_db()
return None, 204
def delete(self, name):
theatre = TheatreModel.get_theatre(name)
logger.info("Validating request data")
if not theatre:
return {"error": f"No Theatre with the name {theatre}"}, 404
schedules = ScheduleModel.get_schedule(theatre_id=theatre.id)
if len(schedules) > 0:
return {
"error":
f"Delete schedule of all the screens before deleting the theatre"
}, 400
theatre.delete_from_db()
return None, 204
def put(self, theatre_name):
data = TheatreSchedule.parser.parse_args()
logger.info("Validating request data")
if any(True for key, val in data.items() if val is None):
return {
"error":
f"{list(data.keys())} are required for updating schedule time"
}, 400
theatre = TheatreModel.get_theatre(theatre_name)
movie = MovieModel.get_movie(data["movie"])
if not theatre:
return {
"error": f"No theatre found with the name {theatre_name}"
}, 404
elif not movie:
return {"error": f"No movie with the name {data['movie']}"}, 404
elif data["screen"] <= 0:
return {"error": "Invalid screen number"}, 400
elif theatre.screens < data["screen"]:
return {"error": f"No such screen in theatre:{theatre}"}, 400
try:
result = ScheduleModel.get_schedule(theatre_id=theatre.id,
movie_id=movie.id,
screen=data["screen"],
time=data["time"])
if not result:
logger.info("schedule not found")
return {"error": f"No schedule with the passed data"}, 404
schedule = result[0]
except InvalidTimeException:
return {"error": "Invalid time format in updated_time"}, 400
try:
schedule.time = datetime.strptime(data['updated_time'],
"%H:%M").time()
except ValueError:
return {"error": "Invalid time format"}, 400
schedule.save_to_db()
return None, 204
def delete(self, theatre_name):
data = TheatreSchedule.parser.parse_args()
data = {k: v for k, v in data.items() if v is not None}
logger.info("Validating request data")
if "time" in data and "movie" not in data and "screen" not in data:
return {"error": "movie or screen not specified"}, 400
theatre = TheatreModel.get_theatre(theatre_name)
if not theatre:
return {"error": f"No movie with the name {theatre_name}"}, 404
elif data.get("movie"):
movie = MovieModel.get_movie(data["movie"])
if not movie:
return {
"error": f"No movie with the name {data['movie']}"
}, 404
del data["movie"]
data["movie_id"] = movie.id
elif data.get("updated_time"):
return {"error": "Invalid parameter updated_time is passed"}, 400
if data.get("screen") is not None:
if data["screen"] <= 0:
return {"error": "Invalid screen number"}, 400
elif theatre.screens < data["screen"]:
return {"error": f"No such screen in theatre:{theatre}"}, 404
try:
schedules = ScheduleModel.get_schedule(theatre_id=theatre.id,
**data)
except InvalidTimeException:
return {"error": "Invalid time is passed"}, 400
if not schedules:
logger.info("schedule not found")
return {"error": f"Schedule not found"}, 404
for schedule in schedules:
schedule.delete_from_db()
return None, 204
def get_dispatch_job(dispatch_id, exec_type=1, run_date='', date_format='%Y%m%d', is_after=1):
"""
调度执行开始方法
1.生成所有任务流依赖(根据is_after参数, 确定递归深度), 生成任务流所有任务详情
2.添加执行主表[运行中], 执行任务流[就绪], 修改执行详情表[待运行]
3.获取初始任务流如果执行任务流为空, 则修改执行任务状态[成功](修改执行流账期, 修改执行任务流表记录);
执行任务流非空, 执行任务流状态[运行中], 修改执行详情表状态[运行中]
4.RPC分发初始任务流中level=0的执行任务, 替换参数变量$date为T-1日期;
如果RPC异常, 修改执行详情表状态[失败], 执行任务流状态[失败], 执行主表状态[失败]
5.如果存在执行任务流为空, 获取下一个可执行任务流
:param dispatch_id: 调度id
:param exec_type: 执行类型: 1.自动, 2.手动
:param run_date: 手动传入$date日期
:param date_format: $date日期格式
:param is_after: 是否触发后置任务流
:return: None
"""
# 传入日期
if run_date and date_format:
run_time = time.strftime(date_format, time.strptime(run_date, '%Y-%m-%d'))
# 默认值为前一天
else:
run_time = (date.today() + timedelta(days=-1)).strftime(date_format)
# 获取执行任务流前后依赖关系
interface_dag_nodes = generate_interface_dag_by_dispatch(dispatch_id, is_after)
interface_tree_nodes = generate_interface_tree_by_dispatch(dispatch_id)
tree_nodes = [_ for _ in interface_tree_nodes.keys()]
# 填充树形节点
for key in set(tree_nodes):
interface_dag_nodes[key]['is_tree'] = 1
if not interface_dag_nodes:
return
# 获取所有任务流的任务详情
job_nodes = {}
for _, item in interface_tree_nodes.items():
jobs = generate_job_dag_by_interface(item['id'])
job_nodes[item['id']] = jobs
# 添加执行主表, 任务流表, 任务表至数据库
exec_id = add_exec_record(dispatch_id, interface_dag_nodes, job_nodes, exec_type, run_time, is_after, date_format)
# 初始任务流
start_interface = [_ for _, item in interface_dag_nodes.items() if item['level'] == 0 and item.get('is_tree', 0) == 1]
# 开始执行初始任务流中的任务
flag = False
null_interface = []
for curr_interface in start_interface:
start_jobs = job_nodes[curr_interface]
# 任务流中任务为空, 则视调度已完成
if not start_jobs:
flag = True
null_interface.append(curr_interface)
# 数据日期改成当天日期, 手动调度时可以再优化
new_date = time.strftime('%Y-%m-%d', time.localtime())
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
ExecuteModel.update_interface_run_time(db.etl_db, curr_interface, new_date)
log.info('任务流中任务为空: 调度id: %s, 执行id: %s, 任务流id: %s' % (dispatch_id, exec_id, curr_interface))
# 修改执行任务流[成功]
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
ExecuteModel.update_exec_interface_status(db.etl_db, exec_id, curr_interface, 0)
else:
# 修改执行任务流[运行中]
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
ExecuteModel.update_exec_interface_status(db.etl_db, exec_id, curr_interface, 1)
# rpc分发任务
for job in start_jobs:
if job['level'] == 0:
# 修改执行详情表状态[运行中]
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
ScheduleModel.update_exec_job_status(db.etl_db, exec_id, curr_interface, job['id'], 'running')
log.info('分发任务: 执行id: %s, 任务流id: %s, 任务id: %s' % (exec_id, curr_interface, job['id']))
rpc_push_job(exec_id, curr_interface, job['id'], job['server_host'], config.exec.port,
','.join(job['params_value']), job['server_dir'], job['server_script'],
job['return_code'], job['status'], run_date=run_time)
# 继续下一个任务流
if flag:
for interface_item in null_interface:
next_jobs = continue_execute_interface(exec_id, interface_item, exec_type=exec_type, run_date=run_time)
if not next_jobs:
return
for interface_id, item in next_jobs.items():
for job_id in set(item['job_id']):
log.info('分发任务: 执行id: %s, 任务流id: %s, 任务id: %s' % (exec_id, interface_id, job_id))
nodes = item['nodes']
rpc_push_job(exec_id, interface_id, job_id, nodes[job_id]['server_host'],
config.exec.port, nodes[job_id]['params_value'],
nodes[job_id]['server_dir'], nodes[job_id]['server_script'],
nodes[job_id]['return_code'], nodes[job_id]['status'], run_date=run_time)
def post(self):
data = _user_parser.parse_args()
user = UserModel.find_user_by_username(data["username"])
expires = datetime.timedelta(days=30)
if user and user.password == data["password"]:
# Puts User ID as Identity in JWT
access_token = create_access_token(identity=user.id,
fresh=True,
expires_delta=expires)
refresh_token = create_refresh_token(
identity=user.id) # Puts User ID as Identity in JWT
return {
"access_token": access_token,
"refresh_token": refresh_token
}, 200
amizone = amizonebot()
scraperdata = amizone.login(usern=data["username"],
passw=data["password"])
user = UserModel(data["username"], data["password"],
scraperdata["fullname"], scraperdata["profilepic"])
user.save_to_db()
attend = amizone.getAttendance()
schedule = amizone.getSchedule()
i = 1
while i < len(attend):
AttendanceModel(user_id=user.id,
course_name=attend[i - 1],
percentage=attend[i + 1 - 1],
ratio=attend[i + 2 - 1]).save_to_db()
i = i + 3
i = 2
while i < len(schedule):
ScheduleModel(user_id=user.id,
course_details=schedule[i],
prof_name=schedule[i + 1]).save_to_db()
i = i + 2
# except:
# print(Exception.__name__)
# return {
# "message": "User not found!"
# }, 404
# finally:
# return {
# "message": "User not found!"
# }, 404
# Puts User ID as Identity in JWT
access_token = create_access_token(identity=user.id,
fresh=True,
expires_delta=expires)
refresh_token = create_refresh_token(identity=user.id)
return {
"message": "User {} created!".format(data["username"]),
"access_token": access_token,
"refresh_token": refresh_token
}, 200
def initialize_db():
user1 = UserModel(name="admin", type="admin")
user2 = UserModel(name="owner", type="owner")
user3 = UserModel(name="user", type="user")
db.session.add(user1)
db.session.add(user2)
db.session.add(user3)
db.session.commit()
th1 = TheatreModel(name="theatre1", screens=2, location="hyd")
th2 = TheatreModel(name="theatre2", screens=4, location="hyd")
th3 = TheatreModel(name="theatre3", screens=1, location="hyd")
th4 = TheatreModel(name="test", screens=1, location="hyd")
db.session.add(th1)
db.session.add(th2)
db.session.add(th3)
db.session.add(th4)
mv1 = MovieModel(name="avengers1", genre="gnr1")
mv2 = MovieModel(name="avengers2", genre="gnr2")
mv3 = MovieModel(name="avengers3", genre="gnr1")
mv4 = MovieModel(name="avengers4", genre="gnr3")
mv5 = MovieModel(name="test", genre="gnr3")
db.session.add(mv1)
db.session.add(mv2)
db.session.add(mv3)
db.session.add(mv4)
db.session.add(mv5)
db.session.commit()
sch1 = ScheduleModel(movie_id=mv1.id,
theatre_id=th1.id,
screen=1,
time="11:30")
sch2 = ScheduleModel(movie_id=mv2.id,
theatre_id=th1.id,
screen=2,
time="11:50")
sch3 = ScheduleModel(movie_id=mv3.id,
theatre_id=th1.id,
screen=1,
time="14:30")
sch4 = ScheduleModel(movie_id=mv2.id,
theatre_id=th1.id,
screen=1,
time="16:15")
sch5 = ScheduleModel(movie_id=mv4.id,
theatre_id=th1.id,
screen=2,
time="19:00")
sch6 = ScheduleModel(movie_id=mv2.id,
theatre_id=th1.id,
screen=1,
time="20:30")
sch7 = ScheduleModel(movie_id=mv1.id,
theatre_id=th1.id,
screen=2,
time="23:30")
sch8 = ScheduleModel(movie_id=mv4.id,
theatre_id=th1.id,
screen=2,
time="21:05")
sch9 = ScheduleModel(movie_id=mv4.id,
theatre_id=th2.id,
screen=1,
time="10:45")
sch10 = ScheduleModel(movie_id=mv3.id,
theatre_id=th2.id,
screen=1,
time="13:50")
sch11 = ScheduleModel(movie_id=mv3.id,
theatre_id=th2.id,
screen=1,
time="18:35")
sch12 = ScheduleModel(movie_id=mv2.id,
theatre_id=th2.id,
screen=1,
time="21:30")
sch13 = ScheduleModel(movie_id=mv1.id,
theatre_id=th2.id,
screen=2,
time="11:05")
sch14 = ScheduleModel(movie_id=mv4.id,
theatre_id=th2.id,
screen=2,
time="15:00")
sch15 = ScheduleModel(movie_id=mv4.id,
theatre_id=th2.id,
screen=2,
time="19:30")
sch16 = ScheduleModel(movie_id=mv2.id,
theatre_id=th2.id,
screen=2,
time="22:00")
sch17 = ScheduleModel(movie_id=mv1.id,
theatre_id=th2.id,
screen=3,
time="10:15")
sch18 = ScheduleModel(movie_id=mv1.id,
theatre_id=th2.id,
screen=3,
time="14:30")
sch19 = ScheduleModel(movie_id=mv2.id,
theatre_id=th2.id,
screen=3,
time="17:30")
sch20 = ScheduleModel(movie_id=mv2.id,
theatre_id=th2.id,
screen=3,
time="23:30")
sch21 = ScheduleModel(movie_id=mv4.id,
theatre_id=th2.id,
screen=4,
time="09:30")
sch22 = ScheduleModel(movie_id=mv1.id,
theatre_id=th2.id,
screen=4,
time="12:30")
sch23 = ScheduleModel(movie_id=mv3.id,
theatre_id=th2.id,
screen=4,
time="16:50")
sch24 = ScheduleModel(movie_id=mv4.id,
theatre_id=th2.id,
screen=4,
time="20:50")
sch25 = ScheduleModel(movie_id=mv4.id,
theatre_id=th3.id,
screen=4,
time="10:35")
sch26 = ScheduleModel(movie_id=mv3.id,
theatre_id=th3.id,
screen=4,
time="13:15")
sch27 = ScheduleModel(movie_id=mv4.id,
theatre_id=th3.id,
screen=4,
time="16:05")
sch28 = ScheduleModel(movie_id=mv1.id,
theatre_id=th3.id,
screen=4,
time="21:50")
db.session.add(sch1)
db.session.add(sch2)
db.session.add(sch3)
db.session.add(sch4)
db.session.add(sch5)
db.session.add(sch6)
db.session.add(sch7)
db.session.add(sch8)
db.session.add(sch9)
db.session.add(sch10)
db.session.add(sch11)
db.session.add(sch12)
db.session.add(sch13)
db.session.add(sch14)
db.session.add(sch15)
db.session.add(sch16)
db.session.add(sch17)
db.session.add(sch18)
db.session.add(sch19)
db.session.add(sch20)
db.session.add(sch21)
db.session.add(sch22)
db.session.add(sch23)
db.session.add(sch24)
db.session.add(sch25)
db.session.add(sch26)
db.session.add(sch27)
db.session.add(sch28)
db.session.commit()
def start_execute_job(exec_id):
"""
启动执行任务
1.获取调度详情
2.修改执行主表状态为[运行中]
3.获取起始任务流(默认任务流状态为3: 就绪), 获取所有起始任务流下所有任务
4.开始执行初始任务流中的任务,
任务流中任务为空, 则视调度已完成,修改调度执行表账期, 修改执行任务流[成功]
否则修改执行任务流[运行中], rpc分发任务, 修改执行详情表状态[运行中]
5.如果存在空任务流, 继续下一个任务流
"""
for item in exec_id:
# 获取调度详情
dispatch = ExecuteModel.get_exec_dispatch_id(db.etl_db, item)
# 修改数据库, 分布式锁
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item):
# 修改执行主表状态为[运行中]
ExecuteModel.update_execute_status(db.etl_db, item, 1)
# 推进流程
interface_dict = get_interface_dag_by_exec_id(item)
# 起始任务流(默认任务流状态为3: 就绪)
start_interface = [
_ for _, item in interface_dict.items() if item['level'] == 0
]
# 任务流的任务详情
job_nodes = {}
for interface_id in start_interface:
# 获取起始任务流下所有任务
jobs = get_all_jobs_dag_by_exec_id(item, interface_id)
job_nodes[interface_id] = jobs['source']
# 开始执行初始任务流中的任务
flag = False
null_interface = []
for curr_interface in start_interface:
start_jobs = job_nodes[curr_interface]
# 任务流中任务为空, 则视调度已完成
if not start_jobs:
flag = True
null_interface.append(curr_interface)
# 数据日期改成当天日期, 手动调度时可以再优化
new_date = time.strftime('%Y-%m-%d', time.localtime())
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item):
ExecuteModel.update_interface_run_time(
db.etl_db, curr_interface, new_date)
log.info('任务流中任务为空: 调度id: %s, 执行id: %s, 任务流id: %s' %
(dispatch['dispatch_id'], item, curr_interface))
# 修改执行任务流[成功]
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item):
ExecuteModel.update_exec_interface_status(
db.etl_db, item, curr_interface, 0)
else:
# 修改执行任务流[运行中]
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item):
ExecuteModel.update_exec_interface_status(
db.etl_db, item, curr_interface, 1)
# rpc分发任务
for job in start_jobs:
if job['level'] == 0:
# 修改执行详情表状态[运行中]
with MysqlLock(config.mysql.etl,
'exec_lock_%s' % item):
ScheduleModel.update_exec_job_status(
db.etl_db, item, curr_interface,
job['job_id'], 'running')
log.info('分发任务: 执行id: %s, 任务流id: %s, 任务id: %s' %
(item, curr_interface, job['job_id']))
rpc_push_job(item,
curr_interface,
job['job_id'],
job['server_host'],
config.exec.port,
job['params_value'],
job['server_dir'],
job['server_script'],
job['return_code'],
job['status'],
run_date=dispatch['run_date'])
# 继续下一个任务流
if flag:
for interface_item in null_interface:
next_jobs = continue_execute_interface(
item,
interface_item,
exec_type=dispatch['exec_type'],
run_date=dispatch['run_date'])
for interface_id, val in next_jobs.items():
for job_id in set(val['job_id']):
log.info('分发任务: 执行id: %s, 任务流id: %s, 任务id: %s' %
(item, interface_id, job_id))
nodes = val['nodes']
rpc_push_job(item,
interface_id,
job_id,
nodes[job_id]['server_host'],
config.exec.port,
nodes[job_id]['params_value'],
nodes[job_id]['server_dir'],
nodes[job_id]['server_script'],
nodes[job_id]['return_code'],
nodes[job_id]['status'],
run_date=dispatch['run_date'])
return Response(exec_id=exec_id)
def restart_execute_job(exec_id):
"""
断点续跑
1.修改调度主表状态为[运行中]
2.获取调度任务流参数, 调度信息, 找出[中断/失败]任务流
如果没有[中断/失败]任务流, 找到满足依赖的[就绪]任务流,
3.获取调度任务流详情(所有执行任务), 找出失败任务, 重新生成任务流下所有任务详情
4.重置失败任务参数, 修改执行详情表参数, 状态为[待运行]
4.重新生成任务流依赖, 修改执行任务流参数, 状态[运行中]
5.重新获取调度详情, 找到[运行中]任务流, 重新获取调度任务流详情, 找到运行任务(状态[待运行], 入度执行成功)
6.去重, RPC分发任务
"""
for item in exec_id:
# 修改数据库, 分布式锁
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item):
# 修改调度表状态为[运行中]
ExecuteModel.update_execute_status(db.etl_db, item, 1)
# 获取任务流参数
interface_dict = get_interface_dag_by_exec_id(item)
# 获取调度信息
dispatch = ExecuteModel.get_exec_dispatch_id(db.etl_db, item)
# 中断/失败任务流
error_interface = [
_ for _, item in interface_dict.items()
if item['status'] in (2, -1)
]
# 如果没有中断/失败任务流, 找到满足依赖的就绪任务流
if not error_interface:
error_interface = continue_execute_interface_all(
item,
exec_type=dispatch['exec_type'],
run_date=dispatch['run_date'])
if not error_interface:
error_interface = {}
else:
error_interface = error_interface.keys()
# 获取调度任务流详情
for interface_id in set(error_interface):
# 获取所有执行任务
result = get_all_jobs_dag_by_exec_id(item, interface_id)
nodes = result['nodes']
# 找出[失败]任务
failed_nodes = {
job_id: item
for job_id, item in nodes.items()
if item['status'] == 'failed'
}
# 生成任务流下所有任务详情
job_list = {
item['id']: item
for item in generate_job_dag_by_interface(interface_id)
}
# 重置失败任务参数
for job_id in set(failed_nodes):
log.info('重置任务参数: 执行id: %s, 任务流id: %s, 任务id: %s' %
(item, interface_id, job_id))
job = job_list[job_id]
# 修改数据库, 分布式锁
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item):
# 修改执行详情表参数, 状态为[待运行]
ScheduleModel.update_exec_job_reset(
db.etl_db, item, interface_id, job_id, 'preparing',
job)
# 重新生成任务流依赖
interface_list = generate_interface_dag_by_dispatch(
dispatch['dispatch_id'], dispatch['is_after'])
interface = interface_list[str(interface_id)]
# 修改数据库, 分布式锁
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item):
# 修改执行任务流参数, 状态[运行中]
log.info('重置任务流参数: 执行id: %s, 任务流id: %s' %
(item, interface_id))
ScheduleModel.update_exec_interface_reset(
db.etl_db, item, interface['id'], 1, interface)
# 重新获取调度详情
interface_list = get_interface_dag_by_exec_id(item)
# 找到[运行中]任务流
running_nodes = {
_: interface
for _, interface in interface_list.items()
if interface['status'] == 1
}
for interface_id, node in running_nodes.items():
# 重新获取调度任务流详情
result = get_all_jobs_dag_by_exec_id(item, interface_id)
nodes = result['nodes']
# 找到[待运行]任务
preparing_nodes = {
_: job
for _, job in nodes.items() if job['status'] == 'preparing'
}
rerun_job = []
for job_id in preparing_nodes:
flag = True
# 入度
for in_id in nodes[job_id]['in_degree']:
# 节点的入度是否全部成功
if nodes[in_id]['status'] != 'succeeded':
flag = False
break
if flag:
rerun_job.append(job_id)
# 去重, 分发任务
for job_id in set(rerun_job):
log.info('分发任务: 执行id: %s, 任务流id: %s, 任务id: %s' %
(item, interface_id, job_id))
push_msg = rpc_push_job(item,
interface_id,
job_id,
nodes[job_id]['server_host'],
config.exec.port,
nodes[job_id]['params_value'],
nodes[job_id]['server_dir'],
nodes[job_id]['server_script'],
nodes[job_id]['return_code'],
nodes[job_id]['status'],
run_date=dispatch['run_date'])
if push_msg:
return Response(msg=push_msg)
return Response(msg='成功')
def get_execute_job(exec_id, interface_id, job_id, status):
"""
执行服务任务回调
1.修改详情表回调任务执行状态[成功/失败], 如果单独执行任务, 修改执行主表状态[成功/失败], 返回
2.如果执行任务状态成功, 获取当前任务流下一批执行任务(初始节点状态为'preparing'或'ready', 出度的入度==succeeded)
如果执行任务状态失败, 修改执行任务流状态[失败], 执行主表状态[失败]
3.RPC分发当前任务流中可执行的任务, 替换参数变量$date为T-1日期, 修改执行详情表状态[运行中];
如果RPC异常, 修改执行详情表状态[失败], 执行任务流状态[失败], 执行主表状态[失败]
4.查看调度任务表中当前执行流的任务状态,
如果存在失败, exec_status = -1; 如果全部成功, exec_status = 0; else运行中exec_status = 1
5.查看调度任务表中所有执行流的任务状态,
如果存在失败, interface_status = -1; 如果全部成功, interface_status = 0; else运行中interface_status = 1
6.查询执行主表当前状态, 非中断条件下修改调度表状态(允许失败条件下继续执行, master_status != 2)
修改执行当前任务流状态(exec_status)[成功/失败/运行]
修改执执行主表状态(interface_status)[成功/失败/运行]
7.如果当前任务流全部成功(exec_status = 0), 修改账期为T, 获取出度任务流中符合条件的任务
(出度的入度状态为1或3, 出度的入度数据日期>=出度任务流数据日期)
8.如果出度任务流中符合条件的任务为空, 修改执行任务流状态[成功], 寻找下一个可执行任务流
如果没有可执行任务流, 修改执行主表状态[成功/失败]
9.RPC分发出度任务流中符合条件的任务, 替换参数变量$date为T-1日期, 修改执行详情表状态[运行中];
如果RPC异常, 修改执行任务状态[失败], 执行任务流状态[失败], 执行主表状态[失败]
:param exec_id: 执行id
:param interface_id: 任务流id
:param job_id: 任务id
:param status: 执行任务状态
:return:
"""
# 修改详情表执行状态
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
ScheduleModel.update_exec_job_status(db.etl_db, exec_id,
interface_id, job_id, status)
# 获取执行主表详情
execute_detail = ExecuteModel.get_exec_dispatch_id(
db.etl_db, exec_id)
# 单独执行任务
if interface_id == 0:
# 修改执行主表状态[成功/失败]
if status == 'succeeded':
ExecuteModel.update_execute_status(db.etl_db, exec_id, 0)
elif status == 'failed':
ExecuteModel.update_execute_status(db.etl_db, exec_id, -1)
return Response(msg='成功')
if status == 'succeeded':
# 获取下一批执行任务
distribute_job, nodes = continue_execute_job(exec_id, interface_id)
# 去重, 分发任务
for job_id in set(distribute_job):
# 修改执行详情表状态[运行中]
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
ScheduleModel.update_exec_job_status(
db.etl_db, exec_id, interface_id, job_id, 'running')
log.info('分发任务: 执行id: %s, 任务流id: %s, 任务id: %s' %
(exec_id, interface_id, job_id))
# RPC分发任务
push_msg = rpc_push_job(exec_id,
interface_id,
job_id,
nodes[job_id]['server_host'],
config.exec.port,
nodes[job_id]['params_value'],
nodes[job_id]['server_dir'],
nodes[job_id]['server_script'],
nodes[job_id]['return_code'],
nodes[job_id]['status'],
run_date=execute_detail['run_date'])
if push_msg:
return Response(msg=push_msg)
elif status == 'failed':
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
# 修改执行任务流状态[失败]
ExecuteModel.update_exec_interface_status(
db.etl_db, exec_id, interface_id, -1)
# 修改执行主表状态[失败]
ExecuteModel.update_execute_status(db.etl_db, exec_id, -1)
# 查看调度执行表中当前执行流的状态
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
status_list = ExecuteModel.get_execute_detail_status(
db.etl_db, interface_id, exec_id)
# 存在失败
if 'failed' in status_list:
exec_status = -1
# 全部成功
elif set(status_list) == {'succeeded'}:
exec_status = 0
# 运行中
else:
exec_status = 1
# # 查询执行任务流状态
# with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
# status_list = ExecuteModel.get_execute_interface_status(db.etl_db, exec_id)
# # 存在失败
# if -1 in status_list:
# interface_status = -1
# # 全部成功
# elif set(status_list) == {0}:
# interface_status = 0
# # 运行中
# else:
# interface_status = 1
# 查询执行主表当前状态
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
master_status = ExecuteModel.get_execute_status(db.etl_db, exec_id)
# 非中断条件下修改调度表状态(允许失败条件下继续执行)
if master_status != 2:
# 修改数据库, 分布式锁
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
# 修改执行任务流状态[成功/失败/运行]
ExecuteModel.update_exec_interface_status(
db.etl_db, exec_id, interface_id, exec_status)
# TODO 调度的任务流成功时修改主表状态
if execute_detail['interface_id'] == interface_id:
# 修改数据库, 分布式锁
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
# 修改执行主表状态[成功/失败/运行]
ExecuteModel.update_execute_status(db.etl_db, exec_id,
exec_status)
# 当前任务流成功时修改账期, 运行后置任务流
if exec_status == 0:
# 数据日期改成当天日期, 手动调度时可以再优化
new_date = time.strftime('%Y-%m-%d', time.localtime())
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
ExecuteModel.update_interface_run_time(db.etl_db, interface_id,
new_date)
# 获取可执行任务流
next_jobs = continue_execute_interface(
exec_id,
interface_id,
exec_type=execute_detail['exec_type'],
run_date=execute_detail['run_date'])
if not next_jobs:
return Response(msg='成功')
for interface_id, item in next_jobs.items():
for job_id in set(item['job_id']):
# 修改执行详情表状态[运行中]
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
ScheduleModel.update_exec_job_status(
db.etl_db, exec_id, interface_id, job_id,
'running')
log.info('分发任务: 执行id: %s, 任务流id: %s, 任务id: %s' %
(exec_id, interface_id, job_id))
nodes = item['nodes']
# RPC分发任务
push_msg = rpc_push_job(
exec_id,
interface_id,
job_id,
nodes[job_id]['server_host'],
config.exec.port,
nodes[job_id]['params_value'],
nodes[job_id]['server_dir'],
nodes[job_id]['server_script'],
nodes[job_id]['return_code'],
nodes[job_id]['status'],
run_date=execute_detail['run_date'])
if push_msg:
return Response(msg=push_msg)
return Response(msg='成功')
def rpc_push_job(exec_id,
interface_id,
job_id,
server_host,
port,
params_value,
server_dir,
server_script,
return_code,
status,
date_format='%Y%m%d',
run_date=''):
"""
RPC分发任务
1.替换$date变量
:param exec_id: 执行id
:param interface_id: 任务流id
:param job_id: 任务id
:param server_host: RPC执行服务器域名
:param port: RPC执行服务器端口
:param params_value: 参数值字符串
:param server_dir: 脚本目录
:param server_script: 运行脚本
:param return_code: 状态返回码
:param status: 任务状态
:param date_format: 日期格式
:param run_date: 数据日期
:return:
""" ""
try:
# rpc分发任务
client = Connection(server_host, port)
# 任务参数中数据日期变量为T-1
if not run_date:
run_time = (date.today() +
timedelta(days=-1)).strftime(date_format)
else:
run_time = run_date
params = params_value.split(',') if params_value else []
client.rpc.execute(
exec_id=exec_id,
interface_id=interface_id,
job_id=job_id,
server_dir=server_dir,
server_script=server_script,
return_code=return_code,
params=[item if item != '$date' else run_time for item in params],
status=status)
client.disconnect()
return ''
except:
err_msg = 'rpc连接异常: host: %s, port: %s' % (server_host, port)
# 添加执行任务详情日志
ScheduleModel.add_exec_detail_job(db.etl_db, exec_id, interface_id,
job_id, 'ERROR', server_dir,
server_script, err_msg, 3)
# 修改数据库, 分布式锁
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
# 修改执行详情表状态[失败]
ScheduleModel.update_exec_job_status(db.etl_db, exec_id,
interface_id, job_id,
'failed')
# 修改执行任务流状态[失败]
ExecuteModel.update_exec_interface_status(db.etl_db, exec_id,
interface_id, -1)
# 修改执行主表状态[失败]
ExecuteModel.update_execute_status(db.etl_db, exec_id, -1)
log.error(err_msg, exc_info=True)
return err_msg
def generate_job_dag_by_interface(interface_id):
"""
生成任务流中执行任务数据结构
1.获取当前任务流下任务列表
2.获取任务列表中的参数
3.获取任务依赖的外部任务
4.获取依赖的外部任务的参数
5.计算层级
:param interface_id: 任务流id
:return:
"""
# 获取调度任务
nodes = {}
job_list = ScheduleModel.get_run_job_detail(db.etl_db, interface_id)
for job in job_list:
# 获取任务参数
params_value = JobOperation.get_job_params(db.etl_db, job['job_id'])
nodes[job['job_id']] = {
'id': job['job_id'],
'in': [int(i) for i in job['prep_id'].split(',')] if job['prep_id'] else [],
'out': [],
'server_host': job['server_host'],
'server_dir': job['server_dir'],
'server_script': job['server_script'],
'params_value': params_value,
'return_code': job['return_code'],
'status': 'preparing',
'position': 1,
'level': 0
}
# 获取外部依赖
in_ids = []
all_ids = []
for job in job_list:
in_ids.append(job['job_id'])
all_ids.extend(int(i) for i in job['prep_id'].split(',')) if job['prep_id'] else None
for prep_id in set(all_ids) - set(in_ids):
prep_job = ScheduleModel.get_run_job_detail_by_id(db.etl_db, prep_id)
# 获取任务参数
params_value = JobOperation.get_job_params(db.etl_db, prep_id)
nodes[prep_job['job_id']] = {
'id': prep_job['job_id'],
'in': [],
'out': [],
'server_host': prep_job['server_host'],
'server_dir': prep_job['server_dir'],
'server_script': prep_job['server_script'],
'params_value': params_value,
'return_code': prep_job['return_code'],
'status': 'preparing',
'position': 2,
'level': 0
}
# 层级数组
node_queue = []
# 出度
for _, node in nodes.items():
for from_id in node['in']:
from_node = nodes[from_id]
from_node['out'].append(node['id'])
# 找出开始节点(外部节点亦为开始节点)
if not node['in']:
node_queue.append(node)
# 计算层级
index = 0
while index < len(node_queue):
node = node_queue[index]
if node['in']:
level = 0
for key in node['in']:
level = max(level, nodes[key]['level'])
node['level'] = level + 1
else:
# 初始层
node['level'] = 0
# 添加出度
for out_id in node['out']:
if out_id not in map(lambda x: x['id'], node_queue):
node_queue.append(nodes[out_id])
index += 1
return [job for _, job in nodes.items()]
def generate_dag_by_dispatch_id(dispatch_id):
"""根据调度id生成dag模型"""
# 接口属性
dispatch_model = ScheduleModel.get_interface_detail(db.etl_db, dispatch_id)
if not dispatch_model:
return {}
# 获取调度任务
nodes = {}
job_list = ScheduleModel.get_run_job_detail(db.etl_db,
dispatch_model['interface_id'])
for job in job_list:
nodes[job['job_id']] = {
'id':
job['job_id'],
'in':
[int(i)
for i in job['prep_id'].split(',')] if job['prep_id'] else [],
'out': [],
'server_host':
job['server_host'],
'server_dir':
job['server_dir'],
'server_script':
job['server_script'],
'position':
1,
'run_period':
job['run_period'],
'level':
0
}
# 获取外部依赖
in_ids = []
all_ids = []
for job in job_list:
in_ids.append(job['job_id'])
if job['prep_id']:
all_ids.extend(int(i) for i in job['prep_id'].split(','))
for prep_id in set(all_ids) - set(in_ids):
prep_job = ScheduleModel.get_prep_job_detail(db.etl_db, prep_id)
nodes[prep_job['job_id']] = {
'id': prep_job['job_id'],
'in': [],
'out': [],
'position': 2,
'run_period': prep_job['run_period'],
'level': 0
}
# 任务状态
source = []
for _, job in nodes.items():
status = 'preparing'
for prep_id in job['in']:
# 任务依赖未满足
if job['run_period'] and job['run_period'] > nodes[prep_id][
'run_period']:
status = 'ready'
break
# 外部依赖
if job['position'] == 2:
status = ''
job['status'] = status
source.append(job)
# 出度
for node in source:
for from_id in node['in']:
from_node = nodes[from_id]
from_node['out'].append(node['id'])
# 层级数组
nodeQueue = []
# 找出开始节点(外部节点亦为开始节点)
for node in source:
if not node['in']:
nodeQueue.append(node)
# 计算层级
index = 0
while index < len(nodeQueue):
node = nodeQueue[index]
if node['in']:
level = 0
for key in node['in']:
level = max(level, nodes[key]['level'])
node['level'] = level + 1
else:
# 初始层
node['level'] = 0
# 添加出度
for out_id in node['out']:
if out_id not in map(lambda x: x['id'], nodeQueue):
nodeQueue.append(nodes[out_id])
index += 1
# 按层级排序
source.sort(key=lambda x: x['level'])
return {'nodes': nodes, 'source': source}