async def send_message(request):
print("Connecting to broker")
while True:
try:
producer = Producer()
break
except NoBrokersAvailable:
print("NO BROKER")
time.sleep(0.5)
message_data = request.form.get('message')
producer.send_messages('my-topic', message_data)
return redirect('/')
async def test_produce_client_errors(consumer, dbconn):
start = time.time()
resources = {'unknown': {'host': 'unknownhost', 'timeout': 3, 'period': 1}}
producer = Producer(resources, kafka_config)
producer_task = asyncio.create_task(producer.start())
consumer_task = asyncio.create_task(consumer.start())
await asyncio.sleep(2)
producer_task.cancel()
consumer_task.cancel()
end = time.time()
metrics = await dbconn.fetch(
'SELECT status, response_time, host, date_part(\'epoch\', timestamp) as timestamp FROM metrics'
)
unknown_metrics = [
metric for metric in metrics
if metric['host'] == resources['unknown']['host']
]
assert len(unknown_metrics) == 2
for metric in metrics:
assert metric['timestamp'] < end
assert metric['timestamp'] > start
assert metric['status'] == 0
async def test_produce_different_timeout(consumer, dbconn):
start = time.time()
resources = {
'google': {
'host': 'https://google.com',
'timeout': 0.1,
'period': 2
},
'tutby': {
'host': 'https://tut.by',
'timeout': 3,
'period': 1
}
}
producer = Producer(resources, kafka_config)
producer_task = asyncio.create_task(producer.start())
consumer_task = asyncio.create_task(consumer.start())
await asyncio.sleep(2)
producer_task.cancel()
consumer_task.cancel()
end = time.time()
metrics = await dbconn.fetch(
'SELECT status, response_time, host, date_part(\'epoch\', timestamp) as timestamp FROM metrics'
)
google_metrics = [
metric for metric in metrics
if metric['host'] == resources['google']['host']
]
tutby_metrics = [
metric for metric in metrics
if metric['host'] == resources['tutby']['host']
]
assert len(google_metrics) == 1
assert len(tutby_metrics) == 2
for metric in metrics:
assert metric['timestamp'] < end
assert metric['timestamp'] > start
for metric in google_metrics:
assert metric['status'] == 0
async def main():
provider = Producer(resources, kafka_config)
await provider.start()
def main(args=None):
"""The main entrypoint for losteyelid.checkscheduler.
:param args: The command line arguments."""
__version__ = "0.1"
if args is None:
args = sys.argv[1:]
LOGGER.info("Started LostEyelid CheckScheduler v" + __version__)
LOGGER.info("Current System Time: " + datetime.datetime.now().isoformat())
producer = Producer()
producer.connect()
cur_check = Check({
'node_id': 0,
'metric_id': 0,
'target': '192.168.122.1',
'check_module': 'dummy_check',
'check_type': 'DummyCheck',
'credential': None,
'parameters': None
})
producer.queue_check(cur_check)
cur_check = Check({
'node_id': 0,
'metric_id': 0,
'target': '192.168.122.1',
'check_module': 'icmp_check',
'check_type': 'ICMPCheck',
'credential': None,
'parameters': None
})
producer.queue_check(cur_check)
cur_check = Check({
'node_id': 0,
'metric_id': 0,
'target': '192.168.122.1',
'check_module': 'command_check',
'check_type': 'CommandCheck',
'credential': {
'username': '******',
'password': '******'
},
'parameters': ['SET']
})
producer.queue_check(cur_check)
cur_check = Check({
'node_id':
0,
'metric_id':
0,
'target':
'192.168.122.1',
'check_module':
'powershell_check',
'check_type':
'PowerShellCheck',
'credential': {
'username': '******',
'password': '******'
},
'parameters': [
'Get-Childitem env:* | Select-Object -Property Name,Value'
' | ConvertTo-Json -Compress'
]
})
producer.queue_check(cur_check)
def init_producer(self):
# generate client ID with pub prefix randomly
self.producer = Producer('producer-'+self.name)
self.producer.start()
class Task(threading.Thread):
def __init__(self,*args, **kwargs):
# print(kwargs['name'])
kwargs['name'] = kwargs['name'].split('/')[kwargs['name'].count('/')].replace('.py','')
if kwargs.get('file') != None:
# print(f"from {(kwargs['file'].replace('.py', '')).replace('/', '.')} import *")
exec(f"from {(kwargs['file'].replace('.py', '')).replace('/','.')} import *")
if kwargs.get('target') == None:
kwargs['target'] = eval(f"run")
else:
kwargs['target'] = eval(f"{kwargs['file'].replace('.py', '')}.{kwargs['target']}")
init_producer = kwargs.get('init_producer')
for my_args in ['file','init_producer']: # 删除原本不属于Thread中的参数
if my_args in kwargs.keys():
del kwargs[my_args]
if not callable(kwargs.get('target')):
raise TypeError("the function must be callable")
self.doc = kwargs['target'].__doc__
self.params = inspect.getargspec(kwargs['target']).args
# for param in self.params:
# kwargs[param] = 1
# print(param)
self.args = args
self.kwargs = kwargs
self.init_info()
self.if_loop = False
self.if_notify = True #可以改成level,按照warn,error,info处理
# self.log = ''
super().__init__(*self.args, **self.kwargs)
# if kwargs.get('init_produce')
if init_producer != False:
self.init_producer()
def init_info(self):
self.success = None
self.exception = None
self.start_time = ''
self.stop_time = ''
self.exc_traceback = ''
def run(self):
if self.if_loop: # 定时任务
while True:
self._is_stopped = False
if datetime.datetime.now() > self.next_run:
try:
self.publish('TaskManager:log',self.name+' run')
self._target(*self._args, **self._kwargs)
# self.success = True
# self.exc_traceback = ''
# self.exception = None
except Exception as e:
self.exception = e
self.success = False
self.exc_traceback = ''.join(traceback.format_exception(*sys.exc_info()))
self.publish('TaskManager:log', self.name + ' failed ' + self.exc_traceback)
if self.if_notify:
self.publish('TaskManager:send_email', self.name + ' failed ' + self.exc_traceback)
finally:
self.stop_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self._schedule_next_run()
time.sleep(1)
else:
self._is_stopped = False
try:
self.publish('TaskManager:log', self.name + ' run')
self._target(*self._args, **self._kwargs)
self.success = True
except Exception as e:
self.exception = e
self.success = False
self.exc_traceback = ''.join(traceback.format_exception(*sys.exc_info()))
self.publish('TaskManager:log', self.name + ' failed ' + self.exc_traceback)
if self.if_notify:
self.publish('TaskManager:send_email', self.exc_traceback)
finally:
self.stop_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
def start(self,args=()):
if self.isAlive():
print(self.name,'任务已经在执行!')
else:
# 一个线程只能运行一次，下一次需要初始化(改写了)
# self.kwargs['args'] = args
# self.__init__(*self.args, **self.kwargs)
self.init_info()
self.start_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(self.name,'任务开始执行!')
self._started._flag = False
super().start()
def restart(self,args=()):
self.stop()
self.start(args=self.kwargs['args'])
def stop(self):
"""raises SystemExit in the context of the given thread, which should
cause the thread to exit silently (unless caught)"""
if self.isAlive() == False:
print(self.name, '停止失败！cause:任务已经停止')
self.publish('TaskManager:log', self.name + ' stop failed')
else:
self.raise_exc(SystemExit)
print(self.name,'任务停止成功!')
self.publish('TaskManrodager:log', self.name + ' stop succeed')
time.sleep(1)
# """" 如果不用sleep函数，restart()会提示：任务已经在执行。因为是stop函数没有执行完成，上一个线程还没有被杀死"""
# self._is_stopped = True
def async_raise(self, tid, exctype):
"""raises the exception, performs cleanup if needed"""
tid = ctypes.c_long(tid)
if not inspect.isclass(exctype):
exctype = type(exctype)
res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
if res == 0:
raise ValueError("invalid thread id")
elif res != 1:
# """if it returns a number greater than one, you're in trouble,
# and you should call it again with exc=NULL to revert the effect"""
ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
raise SystemError("PyThreadState_SetAsyncExc failed")
def get_my_tid(self):
"""determines this (self's) thread id"""
if not self.isAlive():
raise threading.ThreadError("the thread is not active")
# do we have it cached?
if hasattr(self, "_thread_id"):
return self._thread_id
# no, look for it in the _active dict
for tid, tobj in threading._active.items():
if tobj is self:
self._thread_id = tid
return tid
raise AssertionError("could not determine the thread's id")
def raise_exc(self, exctype):
"""raises the given exception type in the context of this thread"""
self.async_raise(self.get_my_tid(), exctype)
@property
def status(self):
if self.isAlive():
return 'running'
else:
return 'stopped'
@property
def info(self):
return {'status':self.status,'success':self.success,'name':self.name,'exception':self.exception,'start_time':self.start_time,'stop_time':self.stop_time}
@property
def all_info(self):
return self.__dict__
def set_loop(self,unit,interval,loop_start_time=None):
'''
:param unit: 如果unit值为指定的星期(1-7),开始时间的日期为下一个最近的这个日子。如果unit为['seconds', 'minutes', 'hours', 'days'],则开始时间的日期为今日
:param interval: unit的数量，当unit值为指定的星期(1-7)时，此参数没有用
:param loop_time: 循环开始的时间（不包含日期）如 10:00:00
:return:
'''
self.unit = unit
self.interval = interval
self.if_loop = True
self.next_run = None
self.loop_start_time = loop_start_time
self._schedule_next_run()
def _schedule_next_run(self):
if self.unit not in ['seconds', 'minutes', 'hours', 'days']+list(weekdays):
raise Exception('Invalid unit')
if self.unit in weekdays:
self.period = datetime.timedelta(**{'days': 7})
else:
self.period = datetime.timedelta(**{self.unit: self.interval})
if self.next_run == None: #第一次计算下次运行时间
now = datetime.datetime.now()
weekday_dates = {}
today = datetime.date.today()
idx = (today.weekday() + 1) % 7
for idx in range(7):
t = today + datetime.timedelta(idx)
weekday_dates[t.weekday()] = today + datetime.timedelta(idx)
# if self.loop_time != None: # 指定开始时间
time_values = [int(v) for v in self.loop_start_time.split(':')]
if len(time_values) == 3:
hour, minute, second = time_values
else:
raise Exception("start_time format is invalid!")
self.loop_start_time = datetime.datetime.now().strftime('%Y-%m-%d') + ' ' + self.loop_start_time
if self.unit not in weekdays:
self.next_run = datetime.datetime(now.year, now.month, now.day, hour, minute, second)
else:
next_run_day = weekday_dates[weekdays.index(self.unit)]
self.next_run = datetime.datetime(next_run_day.year, next_run_day.month, next_run_day.day, hour, minute,second)
# if datetime.datetime.now() > self.next_run: #如果指定时间早于当前时间，需要特殊处理。将下次执行时间移到晚于当前
# if self.unit in weekdays:
# self.period = datetime.timedelta(**{'days': 7})
# else:
# self.period = datetime.timedelta(**{self.unit: self.interval})
# if self.unit in ['seconds', 'minutes', 'hours']:
# self.next_run = datetime.datetime.now() + self.period
# else: # 避免多次运行后，任务运行时间延后
# self.next_run = self.next_run + self.period
# else:#没有指定开始时间，则默认为现在
# if self.unit not in weekdays:
# self.next_run = datetime.datetime(now.year, now.month, now.day, now.hour, now.minute, now.second)
# else:
# next_run_day = weekday_dates[weekdays.index(self.unit)]
# self.next_run = datetime.datetime(next_run_day.year, next_run_day.month, next_run_day.day, now.hour,now.minute, now.second)
# self.loop_time = self.next_run
else:#非第一次计算下次运行时间
if self.unit in ['seconds', 'minutes','hours']:
self.next_run = datetime.datetime.now() + self.period
else:# 避免多次运行后，任务运行时间延后
self.next_run = self.next_run + self.period
self.next_run = datetime.datetime(self.next_run.year, self.next_run.month, self.next_run.day, self.next_run.hour, self.next_run.minute, self.next_run.second)
print('下次运行时间:',self.next_run)
def init_producer(self):
# generate client ID with pub prefix randomly
self.producer = Producer('producer-'+self.name)
self.producer.start()
def publish(self,*args):
# 考虑到有些时候没有init_producer,避免报错，所以封装一下
if hasattr(self,'producer'):
self.producer.publish(*args)
queue_manager = QueueManager(QUEUE_SERVICE_URL)
channel = queue_manager.channel()
channel.queue_declare(queue=QueueManager.QUEUE_NAME,
arguments={'x-max-priority': 255})
parser = argparse.ArgumentParser('main_publisher')
parser.add_argument('-n',
dest='number_of_messages',
default=1,
help='The number of messages to generate',
type=int)
args = parser.parse_args()
producer = Producer()
for i in range(0, args.number_of_messages):
template = random.randint(1, 5)
if template == 1:
payload = producer.produce_confirmation_to_mentor()
elif template == 2:
payload = producer.produce_confirmation_to_mentored()
elif template == 3:
payload = producer.produce_notify_mentor_of_mentored_leaving()
elif template == 4:
payload = producer.produce_notify_subscribed_users_not_selected()
else:
payload = producer.produce_notify_subscribed_users_of_mentoring_canceled(
)
channel.basic_publish(exchange='',
routing_key=QueueManager.QUEUE_NAME,