def validate_and_send_workflow(self, validation):
if validation == True:
if self.workflow.is_scheduled == True:
self.first_job = JobModel.findById(
self.elements[self.data["start"]])
data = self.data
resultCode, resultMessage = self._processHosts(
data["hosts"]['hostsType'], data["hosts"]['host_list'])
if not resultCode:
return {
"error":
"An error occurred inserting the job. " + resultMessage
}, 500
resultCode, resultMessage = self._processTasks(self.first_job)
if not resultCode:
return {
"error":
"An error occurred inserting the job. " + resultMessage
}, 500
self.scheduleJob()
else:
self.sendFirstJob(
JobModel.findById(self.elements[self.data["start"]]))
return True
return False
def get_job_list_all(interface_id):
"""获取所有任务列表"""
if not interface_id:
result = JobModel.get_job_list_all(db.etl_db)
else:
result = JobModel.get_job_list_all_by_interface(
db.etl_db, interface_id)
return Response(result=result)
def lagou_spider():
try:
jd = LagouAPI.search('Python', city='全国') #深圳
for j in jd:
info = LagouAPI.geo_info(j)
print(info)
job = JobModel(info)
job.save()
except Exception as e:
print(e)
def delete_job(job_id, user_id):
"""删除任务"""
# 是否在任务依赖中
prep_count = JobModel.is_alive_job(db.etl_db, job_id)
if prep_count:
abort(
400,
**make_result(status=400, msg='任务存在%s个依赖, 不能删除' % prep_count))
result = JobModel.delete_job(db.etl_db, job_id, user_id)
return Response(result=result)
def post(self):
parser = reqparse.RequestParser()
parser.add_argument(
'title',
type=str,
required=True,
help="This field cannot be blank"
)
parser.add_argument(
'description',
type=str,
required=True,
help="This field cannot be blank"
)
parser.add_argument(
'posted_by',
type=int,
required=True,
help="This field cannot be blank"
)
parser.add_argument(
'status',
type=str,
required=True,
help="This field cannot be blank"
)
parser.add_argument(
'date_posted',
type=lambda x: datetime.strptime(x,'%Y-%m-%dT%H:%M:%S'),
required=True,
help="This field cannot be blank"
)
parser.add_argument(
'deadline',
type=lambda x: datetime.strptime(x,'%Y-%m-%dT%H:%M:%S'),
required=True,
help="This field cannot be blank"
)
data = parser.parse_args()
job_title = data['title']
job_description = data['description']
job_posted_by = data['posted_by']
job_status = data['status']
job_posting_date = data['date_posted']
job_deadline = data['deadline']
if JobModel.find_by_posted_by_and_title(job_posted_by, job_title):
return {
'message': 'Job with Title has been already posted by the recruiter'
}, 400
job = JobModel(job_title, job_description, job_posted_by, job_status, job_posting_date, job_deadline)
job.save_to_db()
return {
'message': 'Job successfully created'
}, 200
def delete_job_many(job_id_arr, user_id):
"""批量删除任务"""
err_mag = []
for item in job_id_arr:
# 是否在任务依赖中
prep_count = JobModel.is_alive_job(db.etl_db, item)
if prep_count:
err_mag.append('任务ID: [%s]存在%s个依赖, 不能删除' % (item, prep_count))
if not err_mag:
condition = '(%s)' % ','.join(str(item) for item in job_id_arr)
JobModel.delete_job_many(db.etl_db, condition, user_id)
return Response(msg=err_mag)
def post(self):
self.parser.add_argument("data", type=str, required=True)
args = self.parser.parse_args()
data = json.loads(args["data"])
obj = JobModel(**data)
try:
obj.save()
except Exception as e:
print(e)
return {"msg": "Insert error!", "data": ""}, 404
return {"data": obj.to_dict(), "msg": ""}
def sendScheduleJob(self):
try:
self.first_job.sendScheduleJob(self.first_job.id)
except:
self.sendFirstJob(
JobModel.findById(self.elements[self.data["start"]]))
return
def get_job_from_info_jobs():
p = "analista"
cidade = 'sao-paulo'
URL = "https://www.infojobs.com.br/vagas-de-emprego-"+ p +"-em-" + cidade + ".aspx"
page = requests.get(URL+p)
soup = BeautifulSoup(page.content,'html.parser')
results = soup.find(id='ctl00_phMasterPage_cGrid_divGrid')
job_elems = results.find_all('div',class_='element-vaga')
job_list:[JobModel] = []
for job_elem in job_elems:
company_elem = job_elem.find('div', class_='vaga-company')
title_elem = job_elem.find('a', class_='vagaTitle')
area_elem = job_elem.find('p',class_='area')
location_elem = job_elem.select('p > span', class_='location2')[1]
desc_elem = job_elem.find('div',class_='vagaDesc')
# url_elem = job_elem.find('div > a', class_='vagaDesc')[0]['href']
title = title_elem['title']
company = company_elem.text.strip()
area = area_elem['title']
location = location_elem['title']
description = desc_elem.text.strip()
new_job = JobModel(title,company,location,description,'url_elem',area)
job_list.append(new_job)
return job_list
def processTriggerAndRule(self, event_id, event_type, column, value,
trigger):
if self.notification_id == 0:
return
if event_type == "job":
self.job = JobModel.findById(event_id)
total = 0
rule_list, notification = self.processTrigger(trigger)
for rule in rule_list:
if event_type == "job":
job_id = event_id
count, rule_name, rule_column, rule_value = self.RuleSatisfied(
job_id, event_type, column, value, rule)
if count == 0:
self.rootLogger.info("Rule not attained")
return
if rule_name == None:
self.rootLogger.info("Event not found")
return
self.rootLogger.info("Rule attained")
total = total + count
informations = {
"element": event_type,
"id": job_id,
"name": rule_name,
"value": rule_value,
"column": rule_column
}
if event_type == "task":
task_id = event_id
count, rule_name, rule_column, rule_value = self.RuleTask(
task_id, event_type, column, value, rule)
if count == 0:
self.rootLogger.info("Rule not attained")
return
if rule_name == None:
self.rootLogger.info("Rule attained")
return
total = total + count
informations = {
"element": event_type,
"id": task_id,
"name": rule_name,
"value": rule_value,
"column": rule_column
}
length = len(rule_list)
if trigger.relation == "all":
if total == length:
self.rootLogger.debug("trigger fired")
self.sendNotification(notification, informations)
if trigger.relation == "any":
if total > 0:
self.rootLogger.debug("trigger fired")
self.sendNotification(notification, informations)
self.rootLogger.info("trigger not fired")
def extractTasks(job_id):
print(job_id)
job = JobModel.findById(job_id)
taskList = TaskModel.find(**{"job_id": job_id})
print(taskList)
applyAbstraction(taskList)
return job, taskList
def get(self, id):
obj = JobModel.get_by_id(id)
if obj:
s = obj.to_dict()
return {"data": s, "msg": ""}
return {"msg": "Item not found!", "data": ""}, 404
def get(self):
jobs = JobModel.find_all()
list_of_jobs = []
for job in jobs:
job_details = json.dumps(job.json(), indent=1, sort_keys=True, default=str)
list_of_jobs.append(json.loads(job_details))
return list_of_jobs
def add_job_detail(job_name, interface_id, job_desc, server_id, server_dir,
job_prep, server_script, user_id):
"""新增任务详情"""
# 新增任务详情
job_id = JobModel.add_job_detail(db.etl_db, job_name, interface_id,
job_desc, server_id, server_dir,
server_script, user_id)
# 新增任务依赖
if job_prep:
data = []
for prep_id in job_prep.split(','):
data.append({
'job_id': job_id,
'prep_id': prep_id,
'user_id': user_id
})
JobModel.add_job_prep(db.etl_db, data)
return Response(job_id=job_id)
def get(self, id):
print("id is " + str(id))
job = JobModel.find_by_id(id)
if job != None:
job_details = json.dumps(job.json(), indent=1, sort_keys=True, default=str)
job_details_res = json.loads(job_details)
return job_details_res,200
else:
return {"error": "Job Not Found"},404
def get_job_params(cursor, job_id):
"""获取任务参数"""
# 获取任务
job = JobModel.get_job_detail(cursor, job_id)
if job['is_deleted']:
abort(400, **make_result(status=400, msg='任务已删除, 不能执行'))
# 获取任务参数
job_params = JobModel.get_job_params_by_job_id(db.etl_db, job_id)
params = []
for item in job_params:
if isinstance(item['source_password'], bytes):
item['source_password'] = item['source_password'].decode(
'utf-8', 'ignore')
# 静态参数
if item['param_type'] == 0:
params.append(item['param_value'])
# SQL参数
elif item['param_type'] == 1:
# 获取SQL参数
result = get_db_data_one(
item['source_type'], item['source_host'],
item['source_port'], item['source_user'],
item['source_password'], item['source_database'],
item['auth_type'], item['param_value'])
if result['flag'] == 0:
params.append(result['result'])
else:
abort(
400,
**make_result(status=400,
msg='获取任务SQL参数错误[ERROR: %s]' %
result['msg']))
# 上下文参数
elif item['param_type'] == 2:
# 任务流名称
if item['param_value'] == '$flow_name':
params.append(job['interface_name'])
# 任务名称
elif item['param_value'] == '$job_name':
params.append(job['job_name'])
# 数据日期(该参数需要传递, 变量代替)
elif item['param_value'] == '$date':
params.append('$date')
return params
def sendJob(self, job_id, host_list):
resultCode, resultMessage = self._processTasks(job_id, host_list)
if not resultCode:
return {
"message":
"An error occurred inserting the job. " + resultMessage
}, 500
job = JobModel.findById(job_id)
job.sendJob(job_id)
self.rootLogger.info('Sent to next job.'.format(job_id))
def execute_job(job_id, run_date, date_format='%Y%m%d'):
"""立即执行任务"""
# 获取任务
job = JobModel.get_job_detail(db.etl_db, job_id)
if job['is_deleted']:
abort(400, **make_result(status=400, msg='任务已删除, 不能执行'))
# 获取任务参数
params = JobOperation.get_job_params(db.etl_db, job_id)
# 传入日期
if run_date and date_format:
run_time = time.strftime(date_format,
time.strptime(run_date, '%Y-%m-%d'))
# 默认值为前一天
else:
run_time = (date.today() +
timedelta(days=-1)).strftime(date_format)
# 添加执行表
exec_id = ExecuteModel.add_execute(db.etl_db, 2, 0, run_time, 0,
'%Y%m%d')
# 添加执行详情表
data = {
'exec_id': exec_id,
'interface_id': 0,
'job_id': job_id,
'in_degree': '',
'out_degree': '',
'server_host': job['server_host'],
'server_dir': job['server_dir'],
'params_value': ','.join(params),
'server_script': job['server_script'],
'return_code': job['return_code'],
'position': 1,
'level': 0,
'status': 'preparing',
'insert_time': int(time.time()),
'update_time': int(time.time())
}
ExecuteModel.add_execute_detail(db.etl_db, data)
# RPC分发任务
push_msg = rpc_push_job(exec_id,
0,
job_id,
job['server_host'],
config.exec.port,
','.join(params),
job['server_dir'],
job['server_script'],
job['return_code'],
'preparing',
run_date=run_time)
if push_msg:
return Response(status=False, msg=push_msg)
else:
return Response(status=True, msg='成功')
def delete(self, id):
obj = JobModel.get_by_id(id)
if obj:
try:
obj.delete()
except Exception as e:
print(e)
return {"msg": "Delete error!", "data": ""}, 404
return {"data": obj.to_dict(), "msg": ""}
return {"msg": "Item not found!", "data": ""}, 404
def get_job_list(job_name, job_index, interface_id, is_deleted, page,
limit):
"""获取任务列表"""
condition = []
if job_name:
condition.append('job_name LIKE "%%%%%s%%%%"' % job_name)
if job_index:
condition.append('job_index IN (%s)' %
','.join('"%s"' % item for item in job_index))
if interface_id:
condition.append('interface_id = %s' % interface_id)
if is_deleted == 1:
condition.append('is_deleted = 0')
elif is_deleted == 2:
condition.append('is_deleted = 1')
condition = 'WHERE ' + ' AND '.join(condition) if condition else ''
result = JobModel.get_job_list(db.etl_db, condition, page, limit)
total = JobModel.get_job_count(db.etl_db, condition)
return Response(result=result, total=total)
def update_job_detail(job_id, interface_id, job_name, job_desc, server_id,
server_dir, server_script, old_prep, job_prep,
user_id, is_deleted):
"""修改任务详情"""
# 修改详情
JobModel.update_job_detail(db.etl_db, job_id, interface_id, job_name,
job_desc, server_id, server_dir,
server_script, user_id, is_deleted)
# 修改任务依赖
old_prep = set() if not old_prep else set(
int(i) for i in old_prep.split(','))
job_prep = set() if not job_prep else set(
int(i) for i in job_prep.split(','))
# 删
del_data = []
for prep_id in old_prep - job_prep:
del_data.append({
'job_id': job_id,
'prep_id': prep_id,
'user_id': user_id
})
if del_data:
JobModel.delete_job_prep(db.etl_db, del_data)
# 增
add_data = []
for prep_id in job_prep - old_prep:
add_data.append({
'job_id': job_id,
'prep_id': prep_id,
'user_id': user_id
})
if add_data:
JobModel.add_job_prep(db.etl_db, add_data)
return Response(job_id=job_id)
def get(self, id):
claims = get_jwt_claims()
print(claims)
application = ApplicationModel.find_by_candidate(id)
job_list_id = []
for appl in application:
job_list_id.append(appl.json()['job'])
jobs_applied = []
for list_of_jobs in job_list_id:
job_list = JobModel.find_by_id(list_of_jobs).json()
jobs_applied.append(job_list)
return jobs_applied
def get_job_list(job_name, start_time, end_time, interface_id, is_deleted,
page, limit):
"""获取任务列表"""
condition = []
if job_name:
condition.append('job_name LIKE "%%%%%s%%%%"' % job_name)
if start_time:
condition.append('insert_time >= %s' % start_time)
if end_time:
condition.append('insert_time <= %s' % end_time)
if interface_id:
condition.append('interface_id = %s' % interface_id)
if is_deleted == 1:
condition.append('is_deleted = 0')
elif is_deleted == 2:
condition.append('is_deleted = 1')
condition = 'WHERE ' + ' AND '.join(condition) if condition else ''
result = JobModel.get_job_list(db.etl_db, condition, page, limit)
total = JobModel.get_job_count(db.etl_db, condition)
return Response(result=result, total=total)
def put(self, id):
obj = JobModel.get_by_id(id)
if obj:
self.parser.add_argument("data", type=str, required=True)
args = self.parser.parse_args()
data = json.loads(args["data"])
try:
if data.get("sql_text"
) and not obj.sql_text == data.get("sql_text"):
obj.update({"is_valid": False})
obj = obj.new_to_save(data)
else:
obj.update(data)
except Exception as e:
print(e)
return {"msg": "Update error!", "data": ""}, 404
return {"data": obj.to_dict(), "msg": ""}
return {"msg": "Item not found!", "data": ""}, 404
def RuleSatisfied(self, event_id, event_type, column, value, rule):
result = 0
if rule["element"] == "job":
rule_column = rule["column"]
rule_operation = rule["operation"]
rule_value = rule["value"]
job_id = event_id
try:
get_job = JobModel.findById(job_id)
except:
return result, None, None, None
job = get_job.__dict__
if rule_column == "status":
event_value = value
else:
event_value = job[rule_column]
job_name = get_job.name
result = self.comparison(event_value, rule_value, rule_operation)
return result, job_name, rule_column, rule_value
return result, None, None, None
def get(self, id):
claims = get_jwt_claims()
print(claims)
if claims['identity'] == id:
application = ApplicationModel.find_by_candidate(id)
job_list_id = []
for appl in application:
job_list_id.append(appl.json()['job'])
jobs_applied = []
for list_of_jobs in job_list_id:
job_list = json.dumps(JobModel.find_by_id(list_of_jobs).json(),
indent=1,
sort_keys=True,
default=str)
jobs_applied.append(json.loads(job_list))
return jobs_applied
else:
return {
'message': 'Unauthorized Access',
'error': 'authorization_required'
}, 401
def execute_job(job_id, user_id):
"""立即执行任务"""
# 获取任务
job = JobModel.get_job_detail(db.etl_db, job_id)
if job.get('is_deleted', 1):
abort(400, **make_result(status=400, msg='任务已删除, 不能执行'))
# 添加执行表
exec_id = ExecuteModel.add_execute(db.etl_db, 2, 0)
# 添加执行详情表
data = {
'exec_id': exec_id,
'job_id': job_id,
'in_degree': '',
'out_degree': '',
'server_host': job['server_host'],
'server_dir': job['server_dir'],
'server_script': job['server_script'],
'position': 1,
'level': 0,
'status': 'preparing'
}
ExecuteModel.add_execute_detail(db.etl_db, data)
# rpc分发任务
client = Connection(job['server_host'], config.exec.port)
try:
client.rpc.execute(exec_id=exec_id,
job_id=job['job_id'],
server_dir=job['server_dir'],
server_script=job['server_script'],
status='preparing')
log.info('分发任务: 执行id: %s, 任务id: %s' % (exec_id, job['job_id']))
return Response(status=True)
except:
log.error('rpc连接异常: host: %s, port: %s' %
(job['server_host'], config.exec.port),
exc_info=True)
return Response(status=False)
def get(self, id):
claims = get_jwt_claims()
id_recruiter = claims['identity']
jobs = JobModel.find_by_posted_by(id_recruiter)
id_exists = False
for job in jobs:
if job.json()['id'] == id:
id_exists = True
if id_exists:
application = ApplicationModel.find_by_job(id)
candidate_list_id = []
for appl in application:
candidate_list_id.append(appl.json()['candidate'])
candidates_applied = []
for id in candidate_list_id:
candidate = UserModel.find_by_id(id).json()
candidates_applied.append(candidate)
return candidates_applied
else:
return {
'message': 'Job Not Associated with the recruiter',
'error': 'unauthorized_access'
}, 401
def add_job_detail(job_name, interface_id, job_desc, job_index, server_id,
server_dir, job_prep, job_params, server_script,
user_id, return_code):
"""新增任务详情"""
# 任务名称查重
if JobModel.get_job_detail_by_name(db.etl_db, job_name):
abort(400, **make_result(status=400, msg='任务名称重复, 已存在数据库中'))
# 新增任务详情
job_id = JobModel.add_job_detail(db.etl_db, job_name, interface_id,
job_desc, job_index, server_id,
server_dir, server_script,
return_code, user_id)
# 新增任务依赖
job_prep = job_prep.split(',') if job_prep else []
data = []
for prep_id in job_prep:
data.append({
'job_id': job_id,
'prep_id': prep_id,
'user_id': user_id,
'insert_time': int(time.time()),
'update_time': int(time.time())
})
JobModel.add_job_prep(db.etl_db, data) if data else None
# 新增任务参数
job_params = job_params.split(',') if job_params else []
data = []
for param_id in job_params:
data.append({
'job_id': job_id,
'param_id': param_id,
'insert_time': int(time.time()),
'update_time': int(time.time()),
'user_id': user_id
})
JobModel.add_job_param(db.etl_db, data) if data else None
return Response(job_id=job_id)
def get_job_detail(job_id):
"""获取任务详情"""
result = JobModel.get_job_detail(db.etl_db, job_id)
return Response(result=result)
