def test_write_to_file(self):
# line = "* * * * * echo `date` @2016-04-17 18:22:56.665851 # echo"
my_cron = CronTab()
job = my_cron.new(command='echo `date`', comment='echo')
job.minute.every(1)
my_cron.write('/tmp/output.tab')
self.assertTrue(True)
def _test_crontab(self, generator):
user_cron = CronTab()
job = user_cron.new(command="/usr/bin/echo")
job.setall(generator.cron.strip())
if job.is_valid() == False:
generator.cron = "%d * * * *" % random.randint(1, 59)
generator.save()
return job.render()
def test_read_from_file(self):
my_cron = CronTab()
my_cron.read('/tmp/output.tab')
cron = my_cron.find_comment('echo')
for result in cron:
print result.last_run
self.assertEqual(result.command, "echo `date`")
self.assertTrue(cron)
def __init__(self, filename=settings.CRON_FILE):
super(CrawlerCronTab, self).__init__()
self.filename = filename
if not os.path.exists(self.filename):
logging.error("The cron file %s is not exist!" % (filename))
if not os.path.exists(os.path.dirname(filename)):
os.mkdir(os.path.dirname(filename))
self.crontab = CronTab(tabfile=filename)
self.cron_timeout = 60
self.exe_number = 0
def test_write_to_file_with_same_comment(self):
my_cron = CronTab()
job = my_cron.new(command='ls -al', comment='ls')
my_cron.write('/tmp/output.tab')
job2 = my_cron.new(command='ls -l', comment='ls')
job2.enable(False)
my_cron.write('/tmp/output.tab')
crons = my_cron.find_comment('ls')
count = 0
for cron in crons:
count += 1
self.assertEqual(count, 2)
class CrawlerCronTab(object):
"""Overwrite python-crontab for myself"""
def __init__(self, filename=settings.CRON_FILE):
super(CrawlerCronTab, self).__init__()
self.filename = filename
if not os.path.exists(self.filename):
logging.error("The cron file %s is not exist!" % (filename))
if not os.path.exists(os.path.dirname(filename)):
os.mkdir(os.path.dirname(filename))
self.crontab = CronTab(tabfile=filename)
self.cron_timeout = 60
self.exe_number = 0
def remove_all_jobs_from_crontab(self):
self.crontab.remove_all()
def remove_offline_jobs_from_crontab(self):
jobs = Job.objects(status=Job.STATUS_OFF)
for job in jobs:
generator = CrawlerTaskGenerator.objects(job=job, status=CrawlerTaskGenerator.STATUS_ON).first()
if not generator:
continue
comment = self._task_generator_cron_comment(job)
self.crontab.remove_all(comment=comment)
def update_online_jobs(self):
jobs = Job.objects(status=Job.STATUS_ON).order_by("+priority")
print "The number of job is %d" % (len(jobs))
for job in jobs:
generator = CrawlerTaskGenerator.objects(job=job).order_by('-add_datetime').first()
if not generator:
continue
if not self._test_save_code(generator):
continue
if not self._test_crontab(generator):
continue
if not self._test_install_crontab(generator):
continue
if generator.status == CrawlerTaskGenerator.STATUS_ON:
continue
generator.status = CrawlerTaskGenerator.STATUS_ON
generator.save()
CrawlerTaskGenerator.objects(job=job, status=CrawlerTaskGenerator.STATUS_ON,
id__ne=generator.id).update(status=CrawlerTaskGenerator.STATUS_OFF)
def save_cron_to_file(self, filename=None):
self.crontab.write(self.filename if not filename else filename)
def _task_generator_cron_comment(self, job):
return "job name:%s with id:%s" % (job.name, job.id)
def _test_install_crontab(self, generator):
comment = self._task_generator_cron_comment(generator.job)
self.crontab.remove_all(comment=comment)
if generator.status == CrawlerTaskGenerator.STATUS_OFF:
return False
command = "GeneratorDispatch('%s').run()" % (str(generator.job.id))
cron = self.crontab.new(command=command, comment=comment)
cron.setall(generator.cron.strip())
return cron.render()
def _test_crontab(self, generator):
user_cron = CronTab()
job = user_cron.new(command="/usr/bin/echo")
job.setall(generator.cron.strip())
if job.is_valid() == False:
generator.cron = "%d * * * *" % random.randint(1, 59)
generator.save()
return job.render()
def _test_save_code(self, generator):
path = generator.product_path()
generator.write_code(path)
return True
def task_generator_install(self):
"""
定期更新所有job的生成器crontab的本地文件信息
"""
# self.remove_offline_jobs_from_crontab()
base_time = time.time()
self.remove_all_jobs_from_crontab()
time1 = time.time()
self.update_online_jobs()
time2 = time.time()
self.save_cron_to_file()
time3 = time.time()
logging.info("TOTAL instal time: %d ms, remove jobs:%d ms, update:%d ms, save:%d ms" %
(int(1000 * (time3 - base_time)), int(1000 * (time1 - base_time)), int(1000 * (time2 - time1)),
int(1000 * (time3 - time2))))
return True
def exec_and_save_current_crons(self):
# 定时任务，到时间强制退出。
timer = threading.Timer(self.cron_timeout, force_exit)
timer.start()
pool = multiprocessing.Pool(multiprocessing.cpu_count())
self.exe_number = 0
for job in self.crontab.crons:
# 获得当前需要运行的cron任务
now = datetime.now()
base = dateutil.parser.parse(str(job.last_run))
slices = job.slices.clean_render()
iters = croniter(slices, base)
next_time = iters.get_next(datetime)
if next_time < now:
try:
command = job.command
comment = job.comment
cron = slices
pool.apply_async(exec_command, (command, comment, cron, ))
self.exe_number += 1
except:
raise
job.set_last_run(now)
logging.info('Waiting for all subprocesses done...')
pool.close()
pool.join()
# 将本次运行的cron任务的上次运行时间写回到tabfile文件中
logging.info('All subprocesses done.')
self.save_cron_to_file()
logging.info('Save to file!')
timer.cancel()
def task_generator_run(self):
"""
每分钟运行例程
"""
self.exec_and_save_current_crons()
return True
def test_clean_render(self):
my_cron = CronTab()
job = my_cron.new(command='echo `date`', comment='echo')
job.minute.every(1)
result = job.slices.clean_render()
self.assertEqual(result, "* * * * *")
def test_change_enable_cron_to_disable(self):
my_cron = CronTab()
my_cron.read('/tmp/output.tab')
crons = my_cron.find_comment('ls')
for cron in crons:
if cron.is_enabled():
cron.enable(False)
my_cron.write('/tmp/output.tab')
test_cron = CronTab()
test_cron.read('/tmp/output.tab')
crons = test_cron.find_comment('ls')
for cron in crons:
self.assertFalse(cron.is_enabled())