def run():
scheduler = BlockingScheduler()
scheduler.add_executor(ThreadPoolExecutor(20))
scheduler.add_jobstore(MemoryJobStore())
scheduler.add_job(process_insights,
'interval',
minutes=2,
max_instances=1,
jitter=20)
scheduler.add_job(mark_insights,
'interval',
minutes=2,
max_instances=1,
jitter=20)
scheduler.add_job(download_product_dataset,
'cron',
day='*',
hour='3',
max_instances=1)
scheduler.add_job(refresh_insights,
'cron',
day='*',
hour='4',
max_instances=1)
scheduler.add_job(generate_insights,
'cron',
day='*',
hour='4',
minute=15,
max_instances=1)
scheduler.add_listener(exception_listener, EVENT_JOB_ERROR)
scheduler.start()
def run():
scheduler = BlockingScheduler()
scheduler.add_executor(ThreadPoolExecutor(20))
scheduler.add_jobstore(MemoryJobStore())
scheduler.add_job(
process_insights, "interval", minutes=2, max_instances=1, jitter=20
)
scheduler.add_job(mark_insights, "interval", minutes=2, max_instances=1, jitter=20)
scheduler.add_job(save_facet_metrics, "cron", day="*", hour=1, max_instances=1)
scheduler.add_job(
download_product_dataset, "cron", day="*", hour="3", max_instances=1
)
scheduler.add_job(
functools.partial(refresh_insights, with_deletion=True),
"cron",
day="*",
hour="4",
max_instances=1,
)
scheduler.add_job(
generate_insights, "cron", day="*", hour="4", minute=15, max_instances=1
)
scheduler.add_job(
generate_quality_facets,
"cron",
day="*",
hour="5",
minute=25,
max_instances=1,
)
scheduler.add_listener(exception_listener, EVENT_JOB_ERROR)
scheduler.start()
def schedule():
scheduler = BlockingScheduler()
scheduler.add_executor('processpool')
scheduler.add_job(pull_job, 'cron',
day_of_week='mon-sun', hour=21, minute=30, timezone=mountain_time)
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
scheduler.start()
except (KeyboardInterrupt, SystemExit):
pass
def single_scheduler(i):
global job_instance
words = get_words(i)
job_instance = Instance(words, index)
scheduler = BlockingScheduler()
scheduler.add_executor('processpool')
# scheduler.add_job(tick, 'interval', seconds=200)
scheduler.add_job(tick, 'interval', seconds=YConfig.TRACK_SPAN)
try:
scheduler.start()
except (KeyboardInterrupt, SystemExit):
print('process has exit!!!')
scheduler.shutdown()
def interval_example1():
"""
Demonstrates how to schedule a job to be run in a process pool on 3 second intervals.
"""
scheduler = BlockingScheduler()
scheduler.add_executor('processpool')
scheduler.add_job(tick, 'interval', seconds=3)
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
scheduler.start()
except (KeyboardInterrupt, SystemExit):
pass
def main(args):
run_flag = args[1]
bs = BlockingScheduler()
bs.add_executor(ThreadPoolExecutor(5))
system = SystemStatsReader(conf)
elastic = ElasticStatsReader(conf)
if run_flag == 'sys':
add_job_to_scheduler(bs,run_system_only, [system])
elif run_flag == 'elastic':
add_job_to_scheduler(bs, run_elastic_only, [elastic])
elif run_flag == None:
add_job_to_scheduler(bs,run_system_only, [system])
add_job_to_scheduler(bs, run_elastic_only, [elastic])
bs.start()
def main(argv):
global config
global logger
f = open('./config/manager.yaml', mode='rb')
config = yaml.load(f)
f.close()
logging.basicConfig()
logger = logging.getLogger(config['logging']['name'])
logger.setLevel(get_logging_level())
logging.getLogger('apscheduler.scheduler').setLevel(get_logging_level())
scheduler = BlockingScheduler()
scheduler.add_executor(ThreadPoolExecutor(5))
add_scheduler_tasks(scheduler)
scheduler.start()
def main(argv):
global config
global logger
f = open('../config/manager.yaml', mode='rb')
config = yaml.load(f)
f.close()
logging.basicConfig()
logger = logging.getLogger(config['logging']['name'])
logger.setLevel(get_logging_level())
logging.getLogger('apscheduler.scheduler').setLevel(get_logging_level())
scheduler = BlockingScheduler()
scheduler.add_executor(ThreadPoolExecutor(5))
add_scheduler_tasks(scheduler)
scheduler.start()
def schedule(parsed_args: Union[argparse.Namespace, Dict[str, int]]):
"""
schedules the application to automaticly collect data
pass a namespace or dict with one or more of the following keys
do not include () adding s is optional, ie: both 'year' and 'years' is a valid key
year(s), month(s), week(s), day(s), hour(s), minute(s), second(s)
:param parsed_args: Union[argparse.Namespace, Dict[str, int]]
:return: None
"""
config = {
"year": parsed_args.year,
"month": parsed_args.month,
"week": parsed_args.week,
"day": parsed_args.day,
"hour": parsed_args.hour,
"minute": parsed_args.minute,
"second": parsed_args.second
}
if parsed_args.job_type == "interval":
config = {(k + "s"): config[k]
for k in config
if k not in ["year", "month"] and config[k] is not None}
else:
config = {
key: value
for key, value in config.items() if value is not None
}
scheduler = BlockingScheduler()
scheduler.add_executor("processpool")
scheduler.add_job(store_temp,
parsed_args.job_type,
misfire_grace_time=parsed_args.misfire,
**config)
scheduler.start()
class CronJobs:
"""
class Cron Jobs
"""
def __init__(self):
self.scheduler = BlockingScheduler()
self.scheduler.add_executor('processpool')
def addJobs(self, time_in_seconds, job_to_execut):
"""
add jobs
:param time_in_seconds: time interval
:param job_to_execut: execute the job
"""
self.scheduler.add_job(job_to_execut,
'interval',
seconds=time_in_seconds)
def addJobsWithArgs(self, time_in_seconds, job_to_execut, job_args=None):
"""
add jobs with arguments
:param time_in_seconds: time interval
:param job_to_execut: execute the job
:param job_args: argument's job
"""
self.scheduler.add_job(job_to_execut,
'interval',
seconds=time_in_seconds,
args=job_args)
def startJobs(self):
"""
start jobs
"""
try:
self.scheduler.start()
except (KeyboardInterrupt, SystemExit):
pass
# old_day = datetime.datetime.now() - datetime.timedelta(days=1) #6年前
# old_day = datetime.datetime(2013,5,1,0,0,0)
old_day = None
#fetch_5min_data(old_day, 'bitcoin')
# fetch_5min_data(old_day, 'bitcoin')
fetch_sometime_data('bitcoin', 1, 5, from_date=old_day)
# fetch_sometime_data('bitcoin', 7, 15, from_date=old_day) #7天
# fetch_sometime_data('bitcoin', 30, 60, from_date=old_day) #30天
# fetch_sometime_data('bitcoin', 90, 120, from_date=old_day) #3个月
day_1year_ago = format_today - datetime.timedelta(days=365)
print(format_today, day_1year_ago)
# fetch_sometime_data('bitcoin', 365, 1440, from_date=day_1year_ago) #1年
fetch_sometime_data('bitcoin', 365*4, 2880, from_date=old_day) #全部
if __name__ == '__main__':
scheduler = BlockingScheduler()
scheduler.add_executor('processpool')
scheduler.add_job(run_once, 'interval', seconds=300)
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
# scheduler.start()
run_once()
except (KeyboardInterrupt, SystemExit):
pass
from apscheduler.schedulers.blocking import BlockingScheduler
import pytz
import datetime
def begin():
create_json.begin()
print("create_json finished")
datax_command.begin()
print("datax finished")
algorithm_newHouse.begin()
print("algorithm_newhouse synchronous finished ")
algorithm_secondHouse.begin()
print("algorithm_secondHouse synchronous finished")
algorithm_phoneDevice.begin()
print("algorithm_phoneDevice synchronous finished")
def begin2():
print("test")
if __name__ == '__main__':
timez = pytz.timezone('Asia/Shanghai')
scheduler = BlockingScheduler(timezone=timez)
scheduler.add_executor('processpool')
scheduler.add_job(begin, 'cron', hour=4, minute=00, second=00)
# scheduler.add_job(begin2, 'interval', seconds=2)
scheduler.start()
# begin()
import datetime
from apscheduler.schedulers.blocking import BlockingScheduler
PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
def log(message: str, file="ip_blacklister.log") -> None:
file = os.path.join(PROJECT_ROOT, file)
with open(file, "a") as f:
f.write(message + "\n")
def run():
log(f"ip blacklister starting to scan access.log @ {datetime.datetime.now()}")
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
log(f"ip blacklister scanned access.log @ {datetime.datetime.now()}\n\n")
if __name__ == '__main__':
scheduler = BlockingScheduler()
scheduler.add_executor("processpool")
scheduler.add_job(run, "cron",
hour=12,
misfire_grace_time=3600,
name=f"scan @ {datetime.datetime.now()}")
log(f"\nStarted ip blacklister @ {datetime.datetime.now()}")
log("ip blacklister will scan access.log at 12:00 local time")
scheduler.start()
def run():
# ensure influxdb database exists
ensure_influx_database()
# This call needs to happen on every start of the scheduler to ensure we're not in
# the state where Robotoff is unable to perform tasks because of missing data.
_update_data()
scheduler = BlockingScheduler()
scheduler.add_executor(ThreadPoolExecutor(20))
scheduler.add_jobstore(MemoryJobStore())
# This job takes all of the newly added automatically-processable insights and sets the process_after field on them,
# indicating when these insights should be auto-applied.
scheduler.add_job(mark_insights,
"interval",
minutes=2,
max_instances=1,
jitter=20)
# This job applies all of the automatically-processable insights that have not been applied yet.
scheduler.add_job(process_insights,
"interval",
minutes=2,
max_instances=1,
jitter=20)
# This job exports daily product metrics for monitoring.
scheduler.add_job(save_facet_metrics,
"cron",
day="*",
hour=1,
max_instances=1)
# This job refreshes data needed to generate insights.
scheduler.add_job(_update_data, "cron", day="*", hour="3", max_instances=1)
# This job updates the product insights state with respect to the latest PO dump by:
# - Deleting non-annotated insights for deleted products and insights that
# are no longer applicable.
# - Updating insight attributes.
scheduler.add_job(
functools.partial(refresh_insights, with_deletion=True),
"cron",
day="*",
hour="4",
max_instances=1,
)
# This job generates category insights using ElasticSearch from the last Product Opener data dump.
scheduler.add_job(generate_insights,
"cron",
day="*",
hour="4",
minute=15,
max_instances=1)
scheduler.add_job(
generate_quality_facets,
"cron",
day="*",
hour="5",
minute=25,
max_instances=1,
)
scheduler.add_listener(exception_listener, EVENT_JOB_ERROR)
scheduler.start()
print("Die Verbindung zu „%s@%s“ ist fehlgeschlagen: %s" % (username, hostname, e.args[1]))
if not tolerant:
exit(1)
except SSHException as e:
print("Allgemeiner SSH-Fehler bei Verbindung zu Host „%s“: %s" % (hostname, e))
if not tolerant:
exit(1)
except gaierror:
print("Der Hostname „%s“ konnte nicht gefunden werden." % hostname)
if not tolerant:
exit(1)
else:
scp = SCPClient(ssh.get_transport())
scheduler = BlockingScheduler()
scheduler.add_executor('threadpool')
if not tolerant: # Programm-Abrruch in verschiedenen Fehlerfällen
scheduler.add_listener(errorListener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
if (probeCronTabExpression == ''):
scheduler.add_job(probe, 'cron', args=[dbConnection, tolerant, maxProbeTries], coalesce=False, second='*/10') # verwende Standard-Intervall, falls keine Cron-Tab-Expr. gesetzt
else:
scheduler.add_job(probe, CronTrigger.from_crontab(probeCronTabExpression), args=[dbConnection, tolerant, maxProbeTries])
if (plotCronTabExpression == ''):
scheduler.add_job(plot, 'cron', args=[dbConnection, diagramPeriod, remotepath, scp, tolerant], coalesce=False, minute='*/1', second=50) # verwende Standard-Intervall, falls keine Cron-Tab-Expr. gesetzt
else:
scheduler.add_job(plot, CronTrigger.from_crontab(plotCronTabExpression), args=[dbConnection, diagramPeriod, remotepath, scp, tolerant])
#probe(tolerant) #Zum Testen! Später entfernen
class JobScheduler(object):
def __init__(self, every=30, unit='second'):
self.mongo = mongopool.get()
self.cursor = self.mongo.get_database('apscheduler').get_collection(
'jobs')
self.every = every
self.unit = unit
self.scheduler = BlockingScheduler(logger=logger)
self.scheduler.configure(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=pytz.timezone('Asia/Saigon'))
self._set_trigger(every, unit)
def _set_trigger(self, every, unit):
now = datetime.now().astimezone(pytz.timezone('Asia/Saigon'))
if unit == 'second':
self.trigger = CronTrigger(second='*/{}'.format(every),
start_date=now)
elif unit == 'minute':
self.trigger = CronTrigger(minute='*/{}'.format(every),
start_date=now)
elif unit == 'hour':
self.trigger = CronTrigger(hour='*/{}'.format(every),
start_date=now)
elif unit == 'day':
self.trigger = CronTrigger(day='*/{}'.format(every),
start_date=now)
else:
raise Exception(message='Unknown time unit')
def add_jobstore(self, jobstore, alias):
self.scheduler.add_jobstore(jobstore, alias)
def add_executor(self, executor, alias):
self.scheduler.add_executor(executor, alias)
def add_job(self,
job_fn,
id='id1',
name='job1',
jobstore='default',
executor='default',
args=None,
kwargs=None):
now = datetime.now().astimezone(pytz.timezone('Asia/Saigon'))
history = list(self.cursor.find({'_id': id}))
if history:
#TODO: process missing jobs
self.cursor.delete_one({'_id': id})
next_run_time = self.trigger.get_next_fire_time(None, now)
if kwargs:
kwargs['run_time'] = next_run_time
else:
kwargs = {'run_time': next_run_time}
self.scheduler.add_job(job_fn,
trigger=self.trigger,
next_run_time=next_run_time,
id=id,
name=name,
jobstore=jobstore,
executor=executor,
args=args,
kwargs=kwargs)
def remove_job(self, id, jobstore='default'):
self.scheduler.remove_job(job_id=id, jobstore=jobstore)
def callback(self, callback_fn, mark=EVENT_ALL):
self.scheduler.add_listener(callback_fn)
def start(self):
mongopool.put(self.mongo)
self.scheduler.start()
def shutdown(self):
self.scheduler.shutdown()
self.scheduler.scheduled_job
