def validate_and_send_workflow(self, validation): if validation == True: if self.workflow.is_scheduled == True: self.first_job = JobModel.findById( self.elements[self.data["start"]]) data = self.data resultCode, resultMessage = self._processHosts( data["hosts"]['hostsType'], data["hosts"]['host_list']) if not resultCode: return { "error": "An error occurred inserting the job. " + resultMessage }, 500 resultCode, resultMessage = self._processTasks(self.first_job) if not resultCode: return { "error": "An error occurred inserting the job. " + resultMessage }, 500 self.scheduleJob() else: self.sendFirstJob( JobModel.findById(self.elements[self.data["start"]])) return True return False
def get_job_list_all(interface_id): """获取所有任务列表""" if not interface_id: result = JobModel.get_job_list_all(db.etl_db) else: result = JobModel.get_job_list_all_by_interface( db.etl_db, interface_id) return Response(result=result)
def lagou_spider(): try: jd = LagouAPI.search('Python', city='全国') #深圳 for j in jd: info = LagouAPI.geo_info(j) print(info) job = JobModel(info) job.save() except Exception as e: print(e)
def delete_job(job_id, user_id): """删除任务""" # 是否在任务依赖中 prep_count = JobModel.is_alive_job(db.etl_db, job_id) if prep_count: abort( 400, **make_result(status=400, msg='任务存在%s个依赖, 不能删除' % prep_count)) result = JobModel.delete_job(db.etl_db, job_id, user_id) return Response(result=result)
def post(self): parser = reqparse.RequestParser() parser.add_argument( 'title', type=str, required=True, help="This field cannot be blank" ) parser.add_argument( 'description', type=str, required=True, help="This field cannot be blank" ) parser.add_argument( 'posted_by', type=int, required=True, help="This field cannot be blank" ) parser.add_argument( 'status', type=str, required=True, help="This field cannot be blank" ) parser.add_argument( 'date_posted', type=lambda x: datetime.strptime(x,'%Y-%m-%dT%H:%M:%S'), required=True, help="This field cannot be blank" ) parser.add_argument( 'deadline', type=lambda x: datetime.strptime(x,'%Y-%m-%dT%H:%M:%S'), required=True, help="This field cannot be blank" ) data = parser.parse_args() job_title = data['title'] job_description = data['description'] job_posted_by = data['posted_by'] job_status = data['status'] job_posting_date = data['date_posted'] job_deadline = data['deadline'] if JobModel.find_by_posted_by_and_title(job_posted_by, job_title): return { 'message': 'Job with Title has been already posted by the recruiter' }, 400 job = JobModel(job_title, job_description, job_posted_by, job_status, job_posting_date, job_deadline) job.save_to_db() return { 'message': 'Job successfully created' }, 200
def delete_job_many(job_id_arr, user_id): """批量删除任务""" err_mag = [] for item in job_id_arr: # 是否在任务依赖中 prep_count = JobModel.is_alive_job(db.etl_db, item) if prep_count: err_mag.append('任务ID: [%s]存在%s个依赖, 不能删除' % (item, prep_count)) if not err_mag: condition = '(%s)' % ','.join(str(item) for item in job_id_arr) JobModel.delete_job_many(db.etl_db, condition, user_id) return Response(msg=err_mag)
def post(self): self.parser.add_argument("data", type=str, required=True) args = self.parser.parse_args() data = json.loads(args["data"]) obj = JobModel(**data) try: obj.save() except Exception as e: print(e) return {"msg": "Insert error!", "data": ""}, 404 return {"data": obj.to_dict(), "msg": ""}
def sendScheduleJob(self): try: self.first_job.sendScheduleJob(self.first_job.id) except: self.sendFirstJob( JobModel.findById(self.elements[self.data["start"]])) return
def get_job_from_info_jobs(): p = "analista" cidade = 'sao-paulo' URL = "https://www.infojobs.com.br/vagas-de-emprego-"+ p +"-em-" + cidade + ".aspx" page = requests.get(URL+p) soup = BeautifulSoup(page.content,'html.parser') results = soup.find(id='ctl00_phMasterPage_cGrid_divGrid') job_elems = results.find_all('div',class_='element-vaga') job_list:[JobModel] = [] for job_elem in job_elems: company_elem = job_elem.find('div', class_='vaga-company') title_elem = job_elem.find('a', class_='vagaTitle') area_elem = job_elem.find('p',class_='area') location_elem = job_elem.select('p > span', class_='location2')[1] desc_elem = job_elem.find('div',class_='vagaDesc') # url_elem = job_elem.find('div > a', class_='vagaDesc')[0]['href'] title = title_elem['title'] company = company_elem.text.strip() area = area_elem['title'] location = location_elem['title'] description = desc_elem.text.strip() new_job = JobModel(title,company,location,description,'url_elem',area) job_list.append(new_job) return job_list
def processTriggerAndRule(self, event_id, event_type, column, value, trigger): if self.notification_id == 0: return if event_type == "job": self.job = JobModel.findById(event_id) total = 0 rule_list, notification = self.processTrigger(trigger) for rule in rule_list: if event_type == "job": job_id = event_id count, rule_name, rule_column, rule_value = self.RuleSatisfied( job_id, event_type, column, value, rule) if count == 0: self.rootLogger.info("Rule not attained") return if rule_name == None: self.rootLogger.info("Event not found") return self.rootLogger.info("Rule attained") total = total + count informations = { "element": event_type, "id": job_id, "name": rule_name, "value": rule_value, "column": rule_column } if event_type == "task": task_id = event_id count, rule_name, rule_column, rule_value = self.RuleTask( task_id, event_type, column, value, rule) if count == 0: self.rootLogger.info("Rule not attained") return if rule_name == None: self.rootLogger.info("Rule attained") return total = total + count informations = { "element": event_type, "id": task_id, "name": rule_name, "value": rule_value, "column": rule_column } length = len(rule_list) if trigger.relation == "all": if total == length: self.rootLogger.debug("trigger fired") self.sendNotification(notification, informations) if trigger.relation == "any": if total > 0: self.rootLogger.debug("trigger fired") self.sendNotification(notification, informations) self.rootLogger.info("trigger not fired")
def extractTasks(job_id): print(job_id) job = JobModel.findById(job_id) taskList = TaskModel.find(**{"job_id": job_id}) print(taskList) applyAbstraction(taskList) return job, taskList
def get(self, id): obj = JobModel.get_by_id(id) if obj: s = obj.to_dict() return {"data": s, "msg": ""} return {"msg": "Item not found!", "data": ""}, 404
def get(self): jobs = JobModel.find_all() list_of_jobs = [] for job in jobs: job_details = json.dumps(job.json(), indent=1, sort_keys=True, default=str) list_of_jobs.append(json.loads(job_details)) return list_of_jobs
def add_job_detail(job_name, interface_id, job_desc, server_id, server_dir, job_prep, server_script, user_id): """新增任务详情""" # 新增任务详情 job_id = JobModel.add_job_detail(db.etl_db, job_name, interface_id, job_desc, server_id, server_dir, server_script, user_id) # 新增任务依赖 if job_prep: data = [] for prep_id in job_prep.split(','): data.append({ 'job_id': job_id, 'prep_id': prep_id, 'user_id': user_id }) JobModel.add_job_prep(db.etl_db, data) return Response(job_id=job_id)
def get(self, id): print("id is " + str(id)) job = JobModel.find_by_id(id) if job != None: job_details = json.dumps(job.json(), indent=1, sort_keys=True, default=str) job_details_res = json.loads(job_details) return job_details_res,200 else: return {"error": "Job Not Found"},404
def get_job_params(cursor, job_id): """获取任务参数""" # 获取任务 job = JobModel.get_job_detail(cursor, job_id) if job['is_deleted']: abort(400, **make_result(status=400, msg='任务已删除, 不能执行')) # 获取任务参数 job_params = JobModel.get_job_params_by_job_id(db.etl_db, job_id) params = [] for item in job_params: if isinstance(item['source_password'], bytes): item['source_password'] = item['source_password'].decode( 'utf-8', 'ignore') # 静态参数 if item['param_type'] == 0: params.append(item['param_value']) # SQL参数 elif item['param_type'] == 1: # 获取SQL参数 result = get_db_data_one( item['source_type'], item['source_host'], item['source_port'], item['source_user'], item['source_password'], item['source_database'], item['auth_type'], item['param_value']) if result['flag'] == 0: params.append(result['result']) else: abort( 400, **make_result(status=400, msg='获取任务SQL参数错误[ERROR: %s]' % result['msg'])) # 上下文参数 elif item['param_type'] == 2: # 任务流名称 if item['param_value'] == '$flow_name': params.append(job['interface_name']) # 任务名称 elif item['param_value'] == '$job_name': params.append(job['job_name']) # 数据日期(该参数需要传递, 变量代替) elif item['param_value'] == '$date': params.append('$date') return params
def sendJob(self, job_id, host_list): resultCode, resultMessage = self._processTasks(job_id, host_list) if not resultCode: return { "message": "An error occurred inserting the job. " + resultMessage }, 500 job = JobModel.findById(job_id) job.sendJob(job_id) self.rootLogger.info('Sent to next job.'.format(job_id))
def execute_job(job_id, run_date, date_format='%Y%m%d'): """立即执行任务""" # 获取任务 job = JobModel.get_job_detail(db.etl_db, job_id) if job['is_deleted']: abort(400, **make_result(status=400, msg='任务已删除, 不能执行')) # 获取任务参数 params = JobOperation.get_job_params(db.etl_db, job_id) # 传入日期 if run_date and date_format: run_time = time.strftime(date_format, time.strptime(run_date, '%Y-%m-%d')) # 默认值为前一天 else: run_time = (date.today() + timedelta(days=-1)).strftime(date_format) # 添加执行表 exec_id = ExecuteModel.add_execute(db.etl_db, 2, 0, run_time, 0, '%Y%m%d') # 添加执行详情表 data = { 'exec_id': exec_id, 'interface_id': 0, 'job_id': job_id, 'in_degree': '', 'out_degree': '', 'server_host': job['server_host'], 'server_dir': job['server_dir'], 'params_value': ','.join(params), 'server_script': job['server_script'], 'return_code': job['return_code'], 'position': 1, 'level': 0, 'status': 'preparing', 'insert_time': int(time.time()), 'update_time': int(time.time()) } ExecuteModel.add_execute_detail(db.etl_db, data) # RPC分发任务 push_msg = rpc_push_job(exec_id, 0, job_id, job['server_host'], config.exec.port, ','.join(params), job['server_dir'], job['server_script'], job['return_code'], 'preparing', run_date=run_time) if push_msg: return Response(status=False, msg=push_msg) else: return Response(status=True, msg='成功')
def delete(self, id): obj = JobModel.get_by_id(id) if obj: try: obj.delete() except Exception as e: print(e) return {"msg": "Delete error!", "data": ""}, 404 return {"data": obj.to_dict(), "msg": ""} return {"msg": "Item not found!", "data": ""}, 404
def get_job_list(job_name, job_index, interface_id, is_deleted, page, limit): """获取任务列表""" condition = [] if job_name: condition.append('job_name LIKE "%%%%%s%%%%"' % job_name) if job_index: condition.append('job_index IN (%s)' % ','.join('"%s"' % item for item in job_index)) if interface_id: condition.append('interface_id = %s' % interface_id) if is_deleted == 1: condition.append('is_deleted = 0') elif is_deleted == 2: condition.append('is_deleted = 1') condition = 'WHERE ' + ' AND '.join(condition) if condition else '' result = JobModel.get_job_list(db.etl_db, condition, page, limit) total = JobModel.get_job_count(db.etl_db, condition) return Response(result=result, total=total)
def update_job_detail(job_id, interface_id, job_name, job_desc, server_id, server_dir, server_script, old_prep, job_prep, user_id, is_deleted): """修改任务详情""" # 修改详情 JobModel.update_job_detail(db.etl_db, job_id, interface_id, job_name, job_desc, server_id, server_dir, server_script, user_id, is_deleted) # 修改任务依赖 old_prep = set() if not old_prep else set( int(i) for i in old_prep.split(',')) job_prep = set() if not job_prep else set( int(i) for i in job_prep.split(',')) # 删 del_data = [] for prep_id in old_prep - job_prep: del_data.append({ 'job_id': job_id, 'prep_id': prep_id, 'user_id': user_id }) if del_data: JobModel.delete_job_prep(db.etl_db, del_data) # 增 add_data = [] for prep_id in job_prep - old_prep: add_data.append({ 'job_id': job_id, 'prep_id': prep_id, 'user_id': user_id }) if add_data: JobModel.add_job_prep(db.etl_db, add_data) return Response(job_id=job_id)
def get(self, id): claims = get_jwt_claims() print(claims) application = ApplicationModel.find_by_candidate(id) job_list_id = [] for appl in application: job_list_id.append(appl.json()['job']) jobs_applied = [] for list_of_jobs in job_list_id: job_list = JobModel.find_by_id(list_of_jobs).json() jobs_applied.append(job_list) return jobs_applied
def get_job_list(job_name, start_time, end_time, interface_id, is_deleted, page, limit): """获取任务列表""" condition = [] if job_name: condition.append('job_name LIKE "%%%%%s%%%%"' % job_name) if start_time: condition.append('insert_time >= %s' % start_time) if end_time: condition.append('insert_time <= %s' % end_time) if interface_id: condition.append('interface_id = %s' % interface_id) if is_deleted == 1: condition.append('is_deleted = 0') elif is_deleted == 2: condition.append('is_deleted = 1') condition = 'WHERE ' + ' AND '.join(condition) if condition else '' result = JobModel.get_job_list(db.etl_db, condition, page, limit) total = JobModel.get_job_count(db.etl_db, condition) return Response(result=result, total=total)
def put(self, id): obj = JobModel.get_by_id(id) if obj: self.parser.add_argument("data", type=str, required=True) args = self.parser.parse_args() data = json.loads(args["data"]) try: if data.get("sql_text" ) and not obj.sql_text == data.get("sql_text"): obj.update({"is_valid": False}) obj = obj.new_to_save(data) else: obj.update(data) except Exception as e: print(e) return {"msg": "Update error!", "data": ""}, 404 return {"data": obj.to_dict(), "msg": ""} return {"msg": "Item not found!", "data": ""}, 404
def RuleSatisfied(self, event_id, event_type, column, value, rule): result = 0 if rule["element"] == "job": rule_column = rule["column"] rule_operation = rule["operation"] rule_value = rule["value"] job_id = event_id try: get_job = JobModel.findById(job_id) except: return result, None, None, None job = get_job.__dict__ if rule_column == "status": event_value = value else: event_value = job[rule_column] job_name = get_job.name result = self.comparison(event_value, rule_value, rule_operation) return result, job_name, rule_column, rule_value return result, None, None, None
def get(self, id): claims = get_jwt_claims() print(claims) if claims['identity'] == id: application = ApplicationModel.find_by_candidate(id) job_list_id = [] for appl in application: job_list_id.append(appl.json()['job']) jobs_applied = [] for list_of_jobs in job_list_id: job_list = json.dumps(JobModel.find_by_id(list_of_jobs).json(), indent=1, sort_keys=True, default=str) jobs_applied.append(json.loads(job_list)) return jobs_applied else: return { 'message': 'Unauthorized Access', 'error': 'authorization_required' }, 401
def execute_job(job_id, user_id): """立即执行任务""" # 获取任务 job = JobModel.get_job_detail(db.etl_db, job_id) if job.get('is_deleted', 1): abort(400, **make_result(status=400, msg='任务已删除, 不能执行')) # 添加执行表 exec_id = ExecuteModel.add_execute(db.etl_db, 2, 0) # 添加执行详情表 data = { 'exec_id': exec_id, 'job_id': job_id, 'in_degree': '', 'out_degree': '', 'server_host': job['server_host'], 'server_dir': job['server_dir'], 'server_script': job['server_script'], 'position': 1, 'level': 0, 'status': 'preparing' } ExecuteModel.add_execute_detail(db.etl_db, data) # rpc分发任务 client = Connection(job['server_host'], config.exec.port) try: client.rpc.execute(exec_id=exec_id, job_id=job['job_id'], server_dir=job['server_dir'], server_script=job['server_script'], status='preparing') log.info('分发任务: 执行id: %s, 任务id: %s' % (exec_id, job['job_id'])) return Response(status=True) except: log.error('rpc连接异常: host: %s, port: %s' % (job['server_host'], config.exec.port), exc_info=True) return Response(status=False)
def get(self, id): claims = get_jwt_claims() id_recruiter = claims['identity'] jobs = JobModel.find_by_posted_by(id_recruiter) id_exists = False for job in jobs: if job.json()['id'] == id: id_exists = True if id_exists: application = ApplicationModel.find_by_job(id) candidate_list_id = [] for appl in application: candidate_list_id.append(appl.json()['candidate']) candidates_applied = [] for id in candidate_list_id: candidate = UserModel.find_by_id(id).json() candidates_applied.append(candidate) return candidates_applied else: return { 'message': 'Job Not Associated with the recruiter', 'error': 'unauthorized_access' }, 401
def add_job_detail(job_name, interface_id, job_desc, job_index, server_id, server_dir, job_prep, job_params, server_script, user_id, return_code): """新增任务详情""" # 任务名称查重 if JobModel.get_job_detail_by_name(db.etl_db, job_name): abort(400, **make_result(status=400, msg='任务名称重复, 已存在数据库中')) # 新增任务详情 job_id = JobModel.add_job_detail(db.etl_db, job_name, interface_id, job_desc, job_index, server_id, server_dir, server_script, return_code, user_id) # 新增任务依赖 job_prep = job_prep.split(',') if job_prep else [] data = [] for prep_id in job_prep: data.append({ 'job_id': job_id, 'prep_id': prep_id, 'user_id': user_id, 'insert_time': int(time.time()), 'update_time': int(time.time()) }) JobModel.add_job_prep(db.etl_db, data) if data else None # 新增任务参数 job_params = job_params.split(',') if job_params else [] data = [] for param_id in job_params: data.append({ 'job_id': job_id, 'param_id': param_id, 'insert_time': int(time.time()), 'update_time': int(time.time()), 'user_id': user_id }) JobModel.add_job_param(db.etl_db, data) if data else None return Response(job_id=job_id)
def get_job_detail(job_id): """获取任务详情""" result = JobModel.get_job_detail(db.etl_db, job_id) return Response(result=result)