def __init__(self):
self.logger = Logger("executor").getlog()
self.aps_log = Logger("apscheduler").getlog()
self.config = ConfigUtil()
self.dboption = DBOption()
self.process_running = {}
self.scheduler = BlockingScheduler()
self.monitor = Monitor()
def __init__(self, configUtil, host, port, username, password, db):
self.config = configUtil
self.host = host
self.port = port
self.db = db
self.username = username
self.password = password
self.logger = Logger("db", configUtil).getlog()
self.pool = PooledDB(
creator=pymysql, # 指定数据库连接驱动
mincached=1, # 连接池中空闲的最多连接数,0和None表示没有限制
maxcached=20, # 连接池允许的最大连接数,0和None表示没有限制
host=self.host,
port=int(self.port),
user=self.username,
passwd=self.password,
db=self.db,
use_unicode=True,
charset="utf8")
def __init__(self):
self.config = ConfigUtil()
self.host = self.config.get("mysql.host")
self.port = self.config.get("mysql.port")
self.db = self.config.get("mysql.db")
self.username = self.config.get("mysql.username")
self.password = self.config.get("mysql.password")
self.logger = Logger("db").getlog()
self.pool = PooledDB(creator=MySQLdb,
mincached=1,
maxcached=20,
host=self.host,
port=int(self.port),
user=self.username,
passwd=self.password,
db=self.db,
use_unicode=True,
charset="utf8")
class DBUtil(object):
def __init__(self):
self.config = ConfigUtil()
self.host = self.config.get("mysql.host")
self.port = self.config.get("mysql.port")
self.db = self.config.get("mysql.db")
self.username = self.config.get("mysql.username")
self.password = self.config.get("mysql.password")
self.logger = Logger("db").getlog()
self.pool = PooledDB(creator=MySQLdb,
mincached=1,
maxcached=20,
host=self.host,
port=int(self.port),
user=self.username,
passwd=self.password,
db=self.db,
use_unicode=True,
charset="utf8")
def get_connection(self):
try:
mysql_con = self.pool.connection()
return mysql_con
except Exception, e:
self.logger.error(e)
for i in range(3):
try:
time.sleep(5)
mysql_con = self.pool.connection()
return mysql_con
except Exception, e:
self.logger.error(e)
self.logger.error("数据库连接异常执行" + str(i + 1) + "次连接")
return None
import os
import requests
import time
import json
import dbutil
from zhconv import convert
from logutil import Logger
import config
logger = Logger("Api.py")
class CityObj:
def __init__(self, id: int, pid: int, city_code, city_name, post_code,
area_code, c_time):
self.id = id
self.pid = pid
self.city_code = city_code
self.city_name = city_name
self.post_code = post_code
self.area_code = area_code
self.c_time = c_time
def set_children(self, children: list):
self.children = children
def get_children(self):
return self.children
def get_id(self):
return self.id
class DBUtil(object):
# 传入需要连接的数据库的名称dbname和待执行的sql语句sql
def __init__(self, configUtil, host, port, username, password, db):
self.config = configUtil
self.host = host
self.port = port
self.db = db
self.username = username
self.password = password
self.logger = Logger("db", configUtil).getlog()
self.pool = PooledDB(
creator=pymysql, # 指定数据库连接驱动
mincached=1, # 连接池中空闲的最多连接数,0和None表示没有限制
maxcached=20, # 连接池允许的最大连接数,0和None表示没有限制
host=self.host,
port=int(self.port),
user=self.username,
passwd=self.password,
db=self.db,
use_unicode=True,
charset="utf8")
def get_connection(self):
try:
mysql_con = self.pool.connection()
return mysql_con
except Exception as e:
self.logger.error(e)
for i in range(3):
try:
time.sleep(5)
mysql_con = self.pool.connection()
return mysql_con
except Exception as e:
self.logger.error(e)
self.logger.error("数据库连接异常执行" + str(i + 1) + "次连接")
sys.exit(1)
def release_connection(self, connection):
try:
connection.close()
except Exception as e:
self.logger.error(e)
self.logger.error("mysql connection 关闭异常")
sys.exit(1)
def query(self, sql):
# def query(self, sql):
# results = ''
# conn = pymysql.connect(host=self.host,
# port=self.port,
# user=self.username,
# passwd=self.password,
# db=self.db)
# cursor = conn.cursor()
# try:
# # 执行SQL语句
# cursor.execute(sql)
# # 获取所有记录列表
# results = cursor.fetchall()
# except Exception as data:
# print('Error: 执行查询失败，%s' % data)
#
# conn.close()
# return results
try:
connection = self.get_connection()
cursor = connection.cursor()
cursor.execute(sql)
rows = cursor.fetchall()
cursor.close()
self.release_connection(connection)
return rows
except Exception as e:
self.logger.error("执行查询:%s 出错" % (e))
sys.exit(1)
def insert_dict_into_table(self, table_name, data_dict):
cols = ','.join(data_dict.keys())
qmarks = ','.join(['%s'] * len(data_dict))
insert_sql = 'insert into %s (%s) values(%s)' % (table_name, cols,
qmarks)
self.insert(insert_sql, data_dict.values())
def insert(self, sql, values):
try:
connection = self.get_connection()
cursor = connection.cursor()
cursor.execute(sql, values)
connection.commit()
cursor.close()
self.release_connection(connection)
except Exception as e:
self.logger.error("执行查询:%s 出错:%s" % (sql, e))
connection.rollback()
sys.exit(1)
finally:
self.release_connection(connection)
def delete(self, sql):
try:
connection = self.get_connection()
cursor = connection.cursor()
cursor.execute(sql)
connection.commit()
cursor.close()
self.release_connection(connection)
except Exception as e:
self.logger.error("执行查询:%s 出错:%s" % (sql, e))
connection.rollback()
sys.exit(1)
finally:
self.release_connection(connection)
class Executor(object):
def __init__(self):
self.logger = Logger("executor").getlog()
self.aps_log = Logger("apscheduler").getlog()
self.config = ConfigUtil()
self.dboption = DBOption()
self.process_running = {}
self.scheduler = BlockingScheduler()
self.monitor = Monitor()
'''
运行 t_etl_job_queue 中Pending状态的job
'''
def run_queue_job_pending(self):
self.logger.info("\n")
self.logger.info("... interval run run_queue_job_pending ....")
try:
self.check_process_state() # 判断已有的进程状态
logpath = self.config.get("job.log.path")
if logpath is None or len(logpath.strip()) == 0:
raise Exception("can't find slave job.log.path")
if not os.path.exists(logpath):
os.makedirs(logpath)
today = DateUtil.get_today()
today_log_dir = logpath + "/" + today
if not os.path.exists(today_log_dir):
os.makedirs(today_log_dir)
queue_job = self.dboption.get_queue_job_pending()
if queue_job is not None:
job_name = queue_job["job_name"]
etl_job = self.dboption.get_job_info(job_name)
job_status = etl_job["job_status"]
job_retry_count = etl_job["retry_count"]
run_number = queue_job["run_number"]
if not self.check_should_run(job_name, job_status,
job_retry_count, run_number):
return
logfile = today_log_dir + "/" + job_name + "_" + today + ".log." + str(
run_number)
bufsize = 0
logfile_handler = open(logfile, 'w', bufsize)
python_bin = CommonUtil.python_bin(self.config)
run_path = project_path + "/bin/" + "runcommand.py"
child = subprocess.Popen(python_bin +
[run_path, "-job", job_name],
stdout=logfile_handler.fileno(),
stderr=subprocess.STDOUT,
shell=False)
pid = child.pid
if pid > 0:
self.logger.info("创建子进程:" + str(pid) + " 运行Job:" +
str(job_name))
code = self.dboption.update_job_running(job_name)
if code != 1:
try:
self.logger.info("更新Job:" + job_name +
" 运行状态为Running失败,停止创建的进程")
self.terminate_process(child, logfile_handler)
except Exception, e:
self.logger.error(e)
self.logger.error("terminate 子进程异常")
logfile_handler.flush()
logfile_handler.close()
else:
self.logger.info("更新Job:" + job_name + " 运行状态Running")
code = self.dboption.update_job_queue_done(
job_name) # FixMe 事物问题
self.logger.info("更新Queue job:" + str(job_name) +
" 状态为Done,影响行数:" + str(code))
if code != 1:
self.logger.error("更新Job Queue job:" + job_name +
" 状态为Done失败")
self.terminate_process(child, logfile_handler)
self.logger.info("重新修改job_name:" + job_name +
" 状态为Pending 等待下次运行")
self.dboption.update_job_pending_from_running(
job_name)
else:
self.process_running[child] = {
"logfile_handler": logfile_handler,
"job_name": job_name,
"pid": pid
}
else:
self.logger.error("启动子进程异常pid:" + str(pid))
logfile_handler.flush()
logfile_handler.close()
else:
def __init__(self):
self.scheduler = BlockingScheduler()
self.logger = Logger("scheduler").getlog()
self.aps_log = Logger("apscheduler").getlog()
self.dboption = DBOption()
self.config = ConfigUtil()
class Scheduler(object):
def __init__(self):
self.scheduler = BlockingScheduler()
self.logger = Logger("scheduler").getlog()
self.aps_log = Logger("apscheduler").getlog()
self.dboption = DBOption()
self.config = ConfigUtil()
def get_timetrigger_job(self, current):
dstring = DateUtil.format_year_second(current)
day = DateUtil.get_time_day(current)
hour = DateUtil.get_time_hour(current)
minute = DateUtil.get_time_minute(current)
week_day = DateUtil.get_week_day(current)
self.logger.info("获取start_hour:" + str(hour) + " start_minute:" + str(minute) + " 运行的 Job")
time_jobs = self.dboption.get_time_trigger_job(hour, minute)
if time_jobs is None or len(time_jobs) == 0:
self.logger.info(dstring + " 没有需要运行的时间触发Job")
return
else:
try:
for job in time_jobs:
job_name = job["job_name"]
trigger_type = job["trigger_type"]
record = 0
should_run = False
if trigger_type == "day": # 每天运行
should_run = True
elif trigger_type == "month": # 每月运行
start_day = job["start_day"]
if int(start_day) == day:
should_run = True
elif trigger_type == "week": # 每周运行
start_day = job["start_day"]
if int(start_day) == week_day:
should_run = True
if should_run:
record = self.dboption.update_trigger_job_pending(current, job_name)
if record == 1:
self.logger.info("更新时间触发Job:" + job_name + " 状态为Pending")
else:
self.logger.error("更新时间触发Job :" + job_name + " 状态为Pending失败")
else:
self.logger.info("时间触发 Job:" + job_name + " 没有对应时间触发执行方式 trigger_type:" + str(trigger_type))
except Exception, e:
self.logger.error(e)
self.logger.error("处理时间触发Job异常")