def __init__(self,
id,
ip,
service_group,
port,
params,
state,
start_time,
db_update_fun=None):
self.id = id
self.ip = ip
self.service_group = service_group
self._port = port
self.params = params
self.state = state
self.start_time = start_time
self.set_hb_info(heartbeat_time="")
self.db_update_fun = db_update_fun
self.tp_service_dic = {}
self.dfp = DirtyFlagProcess(self)
self._control_rpc = None
Jobs().add_interval_job(HEARTBEAT_EXPIRE_TIME, self._heart_beat_expire)
if "db_name" in self.params:
Jobs().add_cron_job(self.__backup, hour=MYSQL_DB_BACKUP_HOUR)
def start_service(self):
"""
启动服务
:return:
"""
args = ArgumentParser().args
try:
Jobs().add_interval_job(UPDATE_INTERVAL, self.update)
if not self.is_sm:
self.adv = ServiceAdvertiser(args.service_id,
self.service_version, args.port)
self.adv.advertise()
check_ports = {}
if "tcp_port" in args:
check_ports['tcp'] = args.tcp_port
if "http_port" in args:
if not args.is_https:
check_ports['http'] = args.http_port
else:
check_ports['https'] = args.http_port
PortChecker(check_ports).start()
self.thread_ls.extend([Jobs()])
self.services(args, self.thread_ls)
logger.warn("start services for %s, args:%s" %
(self.service_type, args))
gevent.joinall([thread.start() for thread in self.thread_ls])
except:
logger.error(traceback.format_exc())
sys.exit(0)
def init_cron_job():
"""
初始化系统cron job
:return:
"""
apply_req = ArgumentParser().args.apply_req
match = ArgumentParser().args.match
apply_pay = ArgumentParser().args.apply_pay
accept_req = ArgumentParser().args.accept_req
apply_wait = ArgumentParser().args.apply_wait
# init
AcceptApplyMatcher().init(AcceptHelpMgr().match_ls,
ApplyHelpMgr().match_ls,
AcceptHelpMgr().do_accept,
ApplyHelpMgr().do_apply,
ServerInfoMgr().make_unique_id)
Jobs().add_cron_job(ApplyReqFilter().filter,
**{apply_req.split('_')[0]: int(apply_req.split('_')[1])})
Jobs().add_cron_job(AcceptApplyMatcher().match,
**{match.split('_')[0]: int(match.split('_')[1])})
if ArgumentParser().args.enable_pay_check:
Jobs().add_cron_job(ApplyPayFilter().filter,
**{apply_pay.split('_')[0]: int(apply_pay.split('_')[1])})
Jobs().add_cron_job(AcceptReqFilter().filter,
**{accept_req.split('_')[0]: int(accept_req.split('_')[1])})
Jobs().add_cron_job(ApplyWaitFilter().filter,
**{apply_wait.split('_')[0]: int(apply_wait.split('_')[1])})
[logger.warn("cron job:%s" % {"cron": cron_job.cron, "func": cron_job.func}) for cron_job in Jobs().cron_job]
def __init__(self,
loader_cls,
expire_second=CACHE_EXPIRE_SECOND,
expire_check_second=CACHE_EXPIRE_CHECK_SECOND):
self.cache_dic = dict()
self.loader_cls = loader_cls
self.expire_second = expire_second
self.expire_check_second = expire_check_second
Jobs().add_interval_job(self.expire_check_second, self._cache_expire)
Jobs().start()
def start_service(self):
"""
启动服务
:return:
"""
args = ArgumentParser().args
try:
Jobs().add_interval_job(UPDATE_INTERVAL, self.update)
if not self.is_sm:
port = {"tcp": args.tcp_port}
port.update({"https": args.http_port} if args.
is_https else {"http": args.http_port})
self.adv = ServiceAdvertiser(self.service_type, port,
self.get_jid(),
self.service_version)
self.adv.advertise()
checker_ls = []
self.add_port_checker(args, checker_ls)
PortChecker(checker_ls).start()
self.services(args, self.thread_ls)
self.__sys_services(args, self.thread_ls)
logger.warn("start services for %s, args:%s" %
(self.service_type, args))
gevent.joinall([thread.start() for thread in self.thread_ls])
except:
logger.error(traceback.format_exc())
sys.exit(0)
def __init__(self, **kwarg):
for attr, default in self.ATTR_DEFAULT_LST:
if attr not in kwarg:
self.__dict__[attr] = default
else:
self.__dict__[attr] = kwarg[attr]
self._conn = None
self._pool = None
# 间断select1 维持连接，不让数据库服务器给断开
Jobs().add_interval_job(SELECT_1_INTERVAL, self.read_db, "select 1")
def __sys_services(self, args, thread_ls):
"""
添加系统服务接口
:param args: 参数变量
:param thread_ls: 现有的服务列表
:return:
"""
tcp_services = filter(lambda thread: isinstance(thread, TcpRpcServer),
thread_ls)
if not tcp_services:
thread_ls.append(TcpRpcServer(args.tcp_port, TcpRpcHandler))
thread_ls.append(HttpRpcServer(args.http_port, args.is_https))
thread_ls.extend([Jobs()])
def start(self):
self.job = Jobs().add_cron_job(self.__backup, hour=MYSQL_DB_BACKUP_HOUR)
def stop(self):
Jobs().remove_job(self.job)
def start(self):
self.job = Jobs().add_interval_job(HEARTBEAT_EXPIRE_TIME,
self.__expire_check)
def __init__(self):
self._access_token = None
self._get()
Jobs().add_interval_job(CLT_ACCESSTOKEN_REFRESH_TIME, self._get)
def start(self):
self.job = Jobs().add_interval_job(self.check_interval, self.check)
def start(self):
"""
开启工作
:return:None
"""
Jobs().add_interval_job(SERVICE_FRESH_INTERVAL, self.pull)
def advertise(self):
return Jobs().add_interval_job(BEAT_INTERVAL, self.beat)
def start(self):
self.job = Jobs().add_interval_job(SERVICE_VERIFY_INTERVAL_SECONDS,
self.__verify)
def start(self):
self.job = Jobs().add_interval_job(self.check_interval, self.check)
class PortChecker(object):
def __init__(self,
port_dic,
ip="localhost",
check_interval=TIMEOUT_INTERVAL):
self.ip = str(ip)
self.port_dic = port_dic
self.check_interval = check_interval
self.checks = []
self.timing_out = False
self.last_connect_dic = {}
self.init_checks()
self.job = None
def _get_checker(self, protocol):
if protocol == "tcp":
return TcpCommonChecker(self.ip, self.port_dic[protocol])
elif protocol == "http":
return HttpCustomChecker(self.ip, self.port_dic[protocol])
elif protocol == "https":
return HttpsCustomChecker(self.ip, self.port_dic[protocol])
return None
def init_checks(self):
for protocol, port in self.port_dic.items():
checker = self._get_checker(protocol)
if not checker:
logger.warn(
"PortChecker::init_checks protocol:%s, port:%s not checker!!!"
% (protocol, port))
continue
self.checks.append(checker)
self.last_connect_dic.setdefault(checker.port, time.time())
def is_timeing_out(self):
return self.timing_out
def start(self):
self.job = Jobs().add_interval_job(self.check_interval, self.check)
def stop(self):
self.job.stop()
def _on_disconnected(self, checker, since_connected):
logger.error('Unable to connect to my port:%s for %s: killing myself' %
(checker.port, since_connected))
os.kill(os.getpid(), 9)
def _on_connected(self):
pass
def __on_ping_success(self, checker):
self.last_connect_dic[checker.port] = time.time()
self.timing_out = False
self._on_connected()
def __on_ping_timeout(self, checker):
self.timing_out = True
since_connected = int(time.time() -
self.last_connect_dic[checker.port])
if since_connected > 2 * TIMEOUT_GRACE:
logger.error('Unable to connect to my port:%s for %s' %
(checker.port, since_connected))
self._on_disconnected(checker, since_connected)
elif since_connected > TIMEOUT_GRACE:
logger.error('Unable to connect to my port:%s for %s' %
(checker.port, since_connected))
else:
logger.warn(
'Unable to connect to my port:%s. Checking again later' %
checker.port)
def check(self):
for checker in self.checks:
try:
assert checker.ping() == PING_RESPONSE
except:
self.__on_ping_timeout(checker)
continue
self.__on_ping_success(checker)
class PortChecker(object):
def __init__(self, port_info, ip="localhost", check_interval=TIMEOUT_INTERVAL):
"""
:param port_ino: 支持以下两个数据格式
1. [{"type":"http/https/tcp/...", "port":"1111"}]
2. {"http/https/tcp/...":1111,,,,,}
:param ip:
:param check_interval:
:return:
"""
self.ip = str(ip)
self.port_ls = port_info if isinstance(port_info, list) else [{"type": k, "port": v} for k, v in port_info.items()]
self.check_interval = check_interval
self.checks = []
self.timing_out = False
self.last_connect_dic = {}
self.init_checks()
self.job = None
def _get_checker(self, type, port):
if type == "tcp":
return TcpCommonChecker(self.ip, port)
elif type == "http":
return HttpCustomChecker(self.ip, port)
elif type == "https":
return HttpsCustomChecker(self.ip, port)
return None
def init_checks(self):
for port_dic in self.port_ls:
checker = self._get_checker(port_dic['type'], port_dic['port'])
if not checker:
logger.warn("PortChecker::init_checks protocol:%s, port:%s not checker!!!" % (port_dic['type'], port_dic['port']))
continue
self.checks.append(checker)
self.last_connect_dic.setdefault(checker.port, time.time())
def is_timeing_out(self):
return self.timing_out
def start(self):
self.job = Jobs().add_interval_job(self.check_interval, self.check)
def stop(self):
self.job.stop()
def _on_disconnected(self, checker, since_connected):
logger.error('Unable to connect to my port:%s for %s: killing myself' % (checker.port, since_connected))
os.kill(os.getpid(), 9)
def _on_connected(self):
pass
def __on_ping_success(self, checker):
self.last_connect_dic[checker.port] = time.time()
self.timing_out = False
self._on_connected()
def __on_ping_timeout(self, checker):
self.timing_out = True
since_connected = int(time.time() - self.last_connect_dic[checker.port])
if since_connected > 2*TIMEOUT_GRACE:
logger.error('Unable to connect to my port:%s for %s' % (checker.port, since_connected))
self._on_disconnected(checker, since_connected)
elif since_connected > TIMEOUT_GRACE:
logger.error('Unable to connect to my port:%s for %s' % (checker.port, since_connected))
else:
logger.warn('Unable to connect to my port:%s. Checking again later'% checker.port )
def check(self):
for checker in self.checks:
try:
assert checker.ping() == PING_RESPONSE
except:
self.__on_ping_timeout(checker)
continue
self.__on_ping_success(checker)
class PortChecker(object):
def __init__(self,
port_info,
ip="localhost",
check_interval=TIMEOUT_INTERVAL):
"""
:param port_ino: 支持以下两个数据格式
1. [{"type":"http/https/tcp/...", "port":"1111"}]
2. {"http/https/tcp/...":1111,,,,,}
:param ip:
:param check_interval:
:return:
"""
self.ip = str(ip)
self.port_ls = port_info if isinstance(port_info, list) else [{
"type": k,
"port": v
} for k, v in port_info.items()]
self.check_interval = check_interval
self.checks = []
self.timing_out = False
self.last_connect_dic = {}
self.init_checks()
self.job = None
def _get_checker(self, type, port):
if type == "tcp":
return TcpCommonChecker(self.ip, port)
elif type == "http":
return HttpCustomChecker(self.ip, port)
elif type == "https":
return HttpsCustomChecker(self.ip, port)
return None
def init_checks(self):
for port_dic in self.port_ls:
checker = self._get_checker(port_dic['type'], port_dic['port'])
if not checker:
logger.warn(
"PortChecker::init_checks protocol:%s, port:%s not checker!!!"
% (port_dic['type'], port_dic['port']))
continue
self.checks.append(checker)
self.last_connect_dic.setdefault(checker.port, time.time())
def is_timeing_out(self):
return self.timing_out
def start(self):
self.job = Jobs().add_interval_job(self.check_interval, self.check)
def stop(self):
self.job.stop()
def _on_disconnected(self, checker, since_connected):
logger.error('Unable to connect to my port:%s for %s: killing myself' %
(checker.port, since_connected))
os.kill(os.getpid(), 9)
def _on_connected(self):
pass
def __on_ping_success(self, checker):
self.last_connect_dic[checker.port] = time.time()
self.timing_out = False
self._on_connected()
def __on_ping_timeout(self, checker):
self.timing_out = True
since_connected = int(time.time() -
self.last_connect_dic[checker.port])
if since_connected > 2 * TIMEOUT_GRACE:
logger.error('Unable to connect to my port:%s for %s' %
(checker.port, since_connected))
self._on_disconnected(checker, since_connected)
elif since_connected > TIMEOUT_GRACE:
logger.error('Unable to connect to my port:%s for %s' %
(checker.port, since_connected))
else:
logger.warn(
'Unable to connect to my port:%s. Checking again later' %
checker.port)
def check(self):
for checker in self.checks:
try:
assert checker.ping() == PING_RESPONSE
except:
self.__on_ping_timeout(checker)
continue
self.__on_ping_success(checker)
