async def main(kafka_topic=KAFKA_TOPIC, sites=SITES, run_total=None):
ssl_context = None
security_protocol = 'PLAINTEXT'
if KAFKA_SSL_CA is not None:
ssl_context = create_ssl_context(cafile=KAFKA_SSL_CA,
certfile=KAFKA_SSL_CERT,
keyfile=KAFKA_SSL_KEY)
security_protocol = 'SSL'
producer = AIOKafkaProducer(bootstrap_servers=KAFKA_HOST,
value_serializer=serializer,
security_protocol=security_protocol,
ssl_context=ssl_context)
msh = Scheduler()
try:
for site in sites:
url = site['url']
log.info(f'Monitoring {url}')
msh.add_job(
CronJob(run_total=run_total,
name=f'check_{url}').every().second.go(
check_site, producer, site, kafka_topic))
await producer.start()
await msh.start()
except Exception as e:
log.error(f'Got error starting scheduler: {e}')
raise e
finally:
await producer.stop()
def run(self):
self.backend = Backend().init()
if not self.backend:
return
if not Auth().init():
return
Launchy.attach_loop(self.loop)
worker = Worker()
self.task_worker = asyncio.ensure_future(worker.run())
backend_worker = BackendWorker()
self.task_backend_worker = asyncio.ensure_future(backend_worker.run())
aptly_worker = AptlyWorker()
self.task_aptly_worker = asyncio.ensure_future(aptly_worker.run())
notification_worker = NotificationWorker()
self.task_notification_worker = asyncio.ensure_future(notification_worker.run())
cfg = Configuration()
daily_cleanup = cfg.aptly.get("daily_cleanup")
if daily_cleanup is False or daily_cleanup == "off" or daily_cleanup == "disabled":
return
if not daily_cleanup:
daily_cleanup = "04:00"
cleanup_sched = Scheduler(locale="en_US")
cleanup_job = CronJob(name='cleanup').every().day.at(daily_cleanup).go(self.cleanup_task)
cleanup_sched.add_job(cleanup_job)
self.task_cron = asyncio.ensure_future(cleanup_sched.start())
app.set_context_functions(MoliorServer.create_cirrina_context, MoliorServer.destroy_cirrina_context)
app.run(self.host, self.port, logger=self.logger, debug=self.debug)
def enable_cron(self, peer_id):
day = int(self.crons[peer_id].day)
hour = self.crons[peer_id].hour
username = self.crons[peer_id].username
job = CronJob(name=peer_id, loop=self.loop).weekday(day).at(hour).go(
self.callback, peer_id, username)
self.scheduler.add_job(job)
def schedule():
print("Initializing schedule", flush=True)
loop = asyncio.get_event_loop()
loop.call_later(
DELAY, scheduler.add_job,
CronJob(name='snapshot', tolerance=3600)
.every(FREQUENCY).second
.go(capture_exceptions, backup),
)
asyncio.create_task(scheduler.start())
async def add_cron_job(self, pre, question_key):
if pre.messageType == "everyday":
time = pre.data.get("time")
time_array = time.split(":")
time = f"{time_array[0]}:{time_array[1]}"
job = CronJob(name=question_key).every().day.at(time).go(
report_cron_status, question_key)
msh.add_job(job)
elif pre.messageType == "interval":
job = CronJob(name=question_key).every(
pre.data.get("second")).second.go(report_cron_status,
question_key)
msh.add_job(job)
elif pre.messageType == "date":
job = CronJob(name=question_key,
tolerance=100).at("2019-01-15 16:12").go(
report_cron_status, question_key)
msh.add_job(job)
elif pre.messageType == "weekday":
time = pre.data.get("time")
time_array = time.split(":")
time = f"{time_array[0]}:{time_array[1]}"
logger.debug(time)
for i in range(0, 5):
job = CronJob(name=question_key).weekday(i).at(time).go(
report_cron_status, question_key)
msh.add_job(job)
elif pre.messageType == "weekend":
time = pre.data.get("time")
time_array = time.split(":")
time = f"{time_array[0]}:{time_array[1]}"
for i in range(5, 7):
job = CronJob(name=question_key).weekday(i).at(time).go(
report_cron_status, question_key)
msh.add_job(job)
elif pre.messageType == "custom":
dates = pre.data.get("date")
time = pre.data.get("time")
time_array = time.split(":")
time = f"{time_array[0]}:{time_array[1]}"
for i in dates:
job = CronJob(name=question_key).weekday(i).at(time).go(
report_cron_status, question_key)
msh.add_job(job)
ee.on(f"{question_key}/state", self.handle_status)
from async_cron.job import CronJob
from async_cron.schedule import Scheduler
from async_cron.job_loader import FileJobLoader
async def test(*args, **kwargs):
print(args, kwargs)
def tt(*args, **kwargs):
print(args, kwargs)
msh = Scheduler(locale="zh_CN")
myjob = CronJob(name='test', run_total=3).every(5).second.go(test, (1, 2, 3),
name=123)
job2 = CronJob(name='exact', tolerance=100).at("2019-01-15 16:12").go(tt, (5),
age=99)
job3 = CronJob(name='very_hour').every().hour.at(":44").go(tt, (5), age=99)
# add timezone infomation using at func, such as : UTC/GMT+08:00, time_shift=8
job3 = CronJob(name='hour').every().hour.at(
":06", time_shift=8).from_hour(18).to_hour(20).go(tt, (5), age=99)
job4 = CronJob(name='minute').every(1).minute.go(tt, (5), age=99)
job5 = CronJob(name='weekday').weekday(2).at("11:18").go(tt, (5), age=99)
job6 = CronJob(name='monthday').monthday(16).at("11:22").go(tt, (5), age=99)
job7 = CronJob(name='monthday').every(5).monthday(16).at("11:22").go(tt, (5),
age=99)
msh.add_job(myjob)
msh.add_job(job2)
from async_cron.schedule import Scheduler
class SchedulerKernel(KernelActor):
def __init__(self, *args, **kwargs):
super(SchedulerKernel, self).__init__(*args, **kwargs)
self.sch = Scheduler()
async def msg_handler(self, msg=None):
await self.sch.start()
async def test(*args, **kwargs):
print(args, kwargs)
def tt(*args, **kwargs):
print(args, kwargs)
myjob = CronJob(name="test").every(5).second.go(test, (1, 2, 3), name=123)
job2 = (CronJob(name="tt",
tolerance=100).every().at("2019-01-15 16:12").go(tt, (5),
age=99))
bb = SchedulerKernel(mail_station=HashMailStation())
bb.sch.add_job(myjob)
bb.sch.add_job(job2)
bb.tell_nowait("hahaha")
bb.start()
import asyncio
import os
from async_cron.job import CronJob
from async_cron.schedule import Scheduler
from pypi_tools.tracker import tracker
import sentry_sdk
scheduler = Scheduler(locale="en_US")
#sentry_sdk.init(os.environ["SENTRY_PATH"])
tracker_job = CronJob(name='track_packages').every(3).minute.go(tracker)
#scheduler.add_job(tracker_job)
if __name__ == '__main__':
# Execute broadcaster
try:
asyncio.get_event_loop().run_until_complete(scheduler.start())
except KeyboardInterrupt:
print('Scheduler exit')
except Exception as e:
sentry_sdk.capture_exception(e)
def __init__(self, bot):
self.bot = bot
job = CronJob(name='superenalotto_extract').every().day.at('00:00').go(self.extract_superenalotto)
#job = CronJob(name='superenalotto_extract').every(30).second.go(self.extract_superenalotto)
self.bot.scheduler.add_job(job)
await verify_error_proxy_task()
await update_squid_task()
@cron_wait
async def fetch_new_proxy_task():
logger.info("run fetch_new_proxy_task")
await spider.run_spider()
await verifier.verify_new_proxy()
# await verify_error_proxy_task()
await update_squid_task()
if __name__ == '__main__':
logger.info("start")
loop = asyncio.get_event_loop()
loop.run_until_complete(update_squid_task())
msh = Scheduler()
msh.add_job(CronJob().every(10).minute.go(verify_ok_proxy_task))
msh.add_job(CronJob().every(30).minute.go(fetch_new_proxy_task))
try:
loop.run_until_complete(asyncio.wait([
msh.start(),
run_api_server(),
]))
loop.run_forever()
except KeyboardInterrupt:
print('exit')