async def main(kafka_topic=KAFKA_TOPIC, sites=SITES, run_total=None):
ssl_context = None
security_protocol = 'PLAINTEXT'
if KAFKA_SSL_CA is not None:
ssl_context = create_ssl_context(cafile=KAFKA_SSL_CA,
certfile=KAFKA_SSL_CERT,
keyfile=KAFKA_SSL_KEY)
security_protocol = 'SSL'
producer = AIOKafkaProducer(bootstrap_servers=KAFKA_HOST,
value_serializer=serializer,
security_protocol=security_protocol,
ssl_context=ssl_context)
msh = Scheduler()
try:
for site in sites:
url = site['url']
log.info(f'Monitoring {url}')
msh.add_job(
CronJob(run_total=run_total,
name=f'check_{url}').every().second.go(
check_site, producer, site, kafka_topic))
await producer.start()
await msh.start()
except Exception as e:
log.error(f'Got error starting scheduler: {e}')
raise e
finally:
await producer.stop()
def run(self):
self.backend = Backend().init()
if not self.backend:
return
if not Auth().init():
return
Launchy.attach_loop(self.loop)
worker = Worker()
self.task_worker = asyncio.ensure_future(worker.run())
backend_worker = BackendWorker()
self.task_backend_worker = asyncio.ensure_future(backend_worker.run())
aptly_worker = AptlyWorker()
self.task_aptly_worker = asyncio.ensure_future(aptly_worker.run())
notification_worker = NotificationWorker()
self.task_notification_worker = asyncio.ensure_future(notification_worker.run())
cfg = Configuration()
daily_cleanup = cfg.aptly.get("daily_cleanup")
if daily_cleanup is False or daily_cleanup == "off" or daily_cleanup == "disabled":
return
if not daily_cleanup:
daily_cleanup = "04:00"
cleanup_sched = Scheduler(locale="en_US")
cleanup_job = CronJob(name='cleanup').every().day.at(daily_cleanup).go(self.cleanup_task)
cleanup_sched.add_job(cleanup_job)
self.task_cron = asyncio.ensure_future(cleanup_sched.start())
app.set_context_functions(MoliorServer.create_cirrina_context, MoliorServer.destroy_cirrina_context)
app.run(self.host, self.port, logger=self.logger, debug=self.debug)
myjob = CronJob(name='test', run_total=3).every(5).second.go(test, (1, 2, 3),
name=123)
job2 = CronJob(name='exact', tolerance=100).at("2019-01-15 16:12").go(tt, (5),
age=99)
job3 = CronJob(name='very_hour').every().hour.at(":44").go(tt, (5), age=99)
# add timezone infomation using at func, such as : UTC/GMT+08:00, time_shift=8
job3 = CronJob(name='hour').every().hour.at(
":06", time_shift=8).from_hour(18).to_hour(20).go(tt, (5), age=99)
job4 = CronJob(name='minute').every(1).minute.go(tt, (5), age=99)
job5 = CronJob(name='weekday').weekday(2).at("11:18").go(tt, (5), age=99)
job6 = CronJob(name='monthday').monthday(16).at("11:22").go(tt, (5), age=99)
job7 = CronJob(name='monthday').every(5).monthday(16).at("11:22").go(tt, (5),
age=99)
msh.add_job(myjob)
msh.add_job(job2)
msh.add_job(job3)
msh.add_job(job4)
msh.add_job(job5)
msh.add_job(job6)
msh.add_job(job7)
# # */1,*,*,*,* test /bin/python,tt.py aa=123,bb=345 1
f_cron = FileJobLoader(name='f_cron',
file_path='t_cron',
log_path='.',
thread=False)
fjob = CronJob(name='fjob', run_total=1).every(1).second.go(f_cron.run, msh)
await verify_error_proxy_task()
await update_squid_task()
@cron_wait
async def fetch_new_proxy_task():
logger.info("run fetch_new_proxy_task")
await spider.run_spider()
await verifier.verify_new_proxy()
# await verify_error_proxy_task()
await update_squid_task()
if __name__ == '__main__':
logger.info("start")
loop = asyncio.get_event_loop()
loop.run_until_complete(update_squid_task())
msh = Scheduler()
msh.add_job(CronJob().every(10).minute.go(verify_ok_proxy_task))
msh.add_job(CronJob().every(30).minute.go(fetch_new_proxy_task))
try:
loop.run_until_complete(asyncio.wait([
msh.start(),
run_api_server(),
]))
loop.run_forever()
except KeyboardInterrupt:
print('exit')
class GtmHubScheduler(threading.Thread):
days_list = {
'0': _('Monday'),
'1': _('Tuesday'),
'2': _('Wednesday'),
'3': _('Thursday'),
'4': _('Friday'),
'5': _('Saturday'),
'6': _('Sunday')
}
def __init__(self, allowed_usernames, callback):
threading.Thread.__init__(self)
self.crons = {}
self.allowed_usernames = allowed_usernames
self.callback = callback
self.loop = asyncio.get_event_loop()
self.scheduler = Scheduler(loop=self.loop)
self.loop.create_task(self.scheduler.start())
self.start()
def run(self):
self.loop.run_forever()
@staticmethod
def is_valid_day(day):
return day in GtmHubScheduler.days_list.keys()
@staticmethod
def is_valid_hour(hour):
try:
time.strptime(hour, '%H:%M')
return True
except ValueError:
return False
def is_valid_username(self, username):
return username in self.allowed_usernames
def has_cron(self, peer_id):
return peer_id in self.crons.keys()
def need_params(self, peer_id):
return self.has_cron(
peer_id) and not self.crons[peer_id].is_configured()
def create_cron(self, peer_id):
if self.has_cron(peer_id):
raise InputException(_('You have a reminder curently'))
self.crons[peer_id] = Cron(peer_id)
def set_value(self, peer_id, value):
if not self.has_cron(peer_id):
raise InputException(_('You have not a reminder to configure'))
if self.crons[peer_id].is_configured():
raise InputException(_('Your reminder is already configured'))
if self.crons[peer_id].day is None:
if not GtmHubScheduler.is_valid_day(value):
raise InputException(_('Invalid day'))
self.crons[peer_id].day = value
elif self.crons[peer_id].hour is None:
if not GtmHubScheduler.is_valid_hour(value):
raise InputException(_('Invalid hour'))
self.crons[peer_id].hour = value
else:
if not self.is_valid_username(value):
raise InputException(
_('Invalid username\nAvailable usernames : {}').format(
'\n' + '\n'.join(self.allowed_usernames)))
self.crons[peer_id].username = value
def enable_cron(self, peer_id):
day = int(self.crons[peer_id].day)
hour = self.crons[peer_id].hour
username = self.crons[peer_id].username
job = CronJob(name=peer_id, loop=self.loop).weekday(day).at(hour).go(
self.callback, peer_id, username)
self.scheduler.add_job(job)
def cancel_cron(self, peer_id):
try:
self.scheduler.del_job(peer_id)
del self.crons[peer_id]
except KeyError:
InputException(_("You don't have a reminder currently"))