def schedule():
"""Schedule jobs from config and launch a blocking scheduler or trigger a job immediately"""
click.echo('Scheduling jobs...')
jobs = get_jobs()
scheduler = BlockingScheduler(timezone=timezone(TIMEZONE))
for job_name in jobs.keys():
try:
job_args, job_kwargs, cron = get_job_execution_info(job_name)
except JobConfException as e:
click.secho(str(e), err=True, fg='red')
continue
except JobFatalException as e:
click.secho(str(e), err=True, fg='red')
return
scheduler.add_job(execute_wrapper,
args=job_args,
kwargs=job_kwargs,
trigger=CronTrigger.from_crontab(cron),
replace_existing=True,
id=job_name)
click.secho(f'- cron "{cron}" set for job "{job_name}".', fg='green')
scheduler.print_jobs()
click.secho('Scheduler started!', fg='green')
click.echo('Press Ctrl+C to exit')
try:
scheduler.start()
except (KeyboardInterrupt, SystemExit):
click.echo('Bye')
def main():
repo_slugs = ['start-jsk/jsk_apc']
gh_repos_handler = GitHubReposHandler(repo_slugs)
scheduler = BlockingScheduler(logger=logger)
scheduler.add_job(gh_repos_handler.send_empty_pr,
trigger='interval', minutes=5)
scheduler.add_job(gh_repos_handler.close_ci_success_empty_pr,
trigger='interval', minutes=5)
scheduler.print_jobs()
scheduler.start()
def apscheduler():
logging.basicConfig()
scheduler = BlockingScheduler()
# Adds new scraping job every 15 minutes
scheduler.add_job(grabAndSave, 'cron', minute='00,15,30,45')
# Begins jobs
scheduler.start()
scheduler.print_jobs()
logging.getLogger('apscheduler').setLevel(logging.DEBUG)
def start_schedule():
# schedule = BackgroundScheduler()
schedule = BlockingScheduler(timezone="Asia/Shanghai")
log = logging.getLogger('apscheduler.executors.default')
log.setLevel(logging.INFO) # DEBUG
fmt = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
h = logging.StreamHandler()
h.setFormatter(fmt)
log.addHandler(h)
if scheduletype.upper() == "INTERVAL":
schedule.add_job(
run,
"interval",
weeks = waitweeks,
days = waitdays,
hours = waithours,
minutes = waitminutes,
seconds = waitseconds,
start_date = options.start_date,
end_date = options.start_date
)
elif scheduletype.upper() == "CRON":
schedule.add_job(
run,
"cron",
year = options.year,
month = options.month,
week = options.week,
day = options.day,
day_of_week = options.dayofweek,
hour = options.hour,
minute = options.minute,
second = options.second,
start_date = options.start_date,
end_date = options.end_date
)
# jobs = schedule.get_jobs()
schedule.add_listener(automation_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
schedule.print_jobs()
schedule.start()
try:
schedule.start()
except (KeyboardInterrupt, SystemExit):
schedule.shutdown()
print('Exit Schedule Job!')
def run_tasks(sql_loc):
"""
Runs the tasks stored in the task database
:param: sql_loc: location of the sql task database
:type: sql_loc: string
: return: None
"""
jobstores = {'default': SQLAlchemyJobStore(url='sqlite:///%s' % sql_loc)}
scheduler = BlockingScheduler(jobstores=jobstores)
scheduler.start()
scheduler.print_jobs()
return None
def init_start_schedule():
scheduler = BlockingScheduler()
#scheduler.shutdown(wait=False)
url = 'sqlite:////home/apps/dbajob.sqlite'
scheduler.add_jobstore('sqlalchemy', url=url, alias='sqlite_js')
scheduler.print_jobs()
scheduler.start()
print 'success!'
scheduler.print_jobs()
def task():
from apscheduler.schedulers.blocking import BlockingScheduler
res_cron_tasks = get_test_cron_tragger_str()
scheduler = BlockingScheduler()
for res_cron in res_cron_tasks:
scheduler.add_job(**res_cron)
# scheduler.add_job(func=alert_gen_polling, args=(config_dict,), trigger='interval', seconds=5)
# scheduler.add_job(func=update_ip_statistics, args=(config_dict,), trigger='interval', seconds=5)
print(scheduler.print_jobs())
scheduler.start()
class scheduler():
"""This Class provide the main scheduler, it consumes the config
and schedules the drivers to run for their respective metrics"""
def __init__(self, loggingLevel=logging.ERROR):
self.logger = logging.getLogger('sensipy')
self.logger.setLevel(loggingLevel)
fh = logging.FileHandler('error.log')
fh.setLevel(logging.ERROR)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
self.logger.addHandler(fh)
self.logger.addHandler(ch)
self.sender = Sender()
executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
job_defaults = {
'coalesce': False,
'max_instances': 3
}
self.queue = Queue(connection=Redis())
self.scheduler = BlockingScheduler(executors=executors, job_defaults=job_defaults)
self.configFile = 'config.json'
def start(self):
"""Runs all the initialization methods, then starts the
scheduler in standalone (blocking) mode"""
self.loadConf()
self.loadDrivers()
self.loadFeeds()
self.runScheduler()
self.scheduler.print_jobs()
self.scheduler.start()
self.printConf("test")
print("scheduler started")
def stop(self):
print("test")
self.scheduler.shutdown()
def loadDrivers(self):
"""Iterates through the Driver instances stored in the config,
and loads corresponding instances into the driver dict"""
self.sources = {}
for source in self.config['sources']:
sourceConf = self.config['sources'][source]
baseClass = sourceConf['baseClass']
self.logger.debug("Loading: " + source +
" instance of: " + baseClass)
sourceArgs = sourceConf['source-config']
self.sources[source] = {}
try:
print(baseClass)
tempModule = import_module('sources.' + baseClass)
"""tempModule = __import__('sources.' + baseClass,
globals(), locals(), [baseClass], -1)
"""
self.sources[source]['source'] = getattr(tempModule, str(
baseClass))(sourceArgs)
except Exception as e:
self.logger.error("exception: " + str(e))
return None
def loadConf(self):
"""Retrieves config from file specified in __init__"""
with open(self.configFile) as f:
self.config = json.load(f)
def printConf(self, args):
print(args)
print(self.config)
def loadFeeds(self):
"""Sets up each metric in it's corresponding driver instance nice name
"""
metrics = self.config['metrics']
for metric in metrics:
metricConf = self.config['metrics'][metric]
metricConf['name'] = metric
source = metricConf['source']['driver']
if 'metrics' not in self.sources[source['name']]:
self.sources[source['name']]['metrics'] = []
self.sources[source['name']]['metrics'].append(metricConf)
def runScheduler(self):
"""Sets up base scheduler interval for each configured
driver instance"""
for source in self.sources:
intervals = [
int(self.sources[source]['metrics'][x]['interval']) for x
in range(0, len(self.sources[source]['metrics']))]
sourceInterval = self.gcd(intervals)
self.sources[source]['sourceInterval'] = sourceInterval
self.logger.debug(self.sources[source]['metrics'])
self.scheduler.add_job(
self.getDriverData, 'interval', args=[
self.sources[source]['metrics']],
seconds=sourceInterval)
def getDriverData(self, metricSet):
"""Gets data from a single driver instance, on the intervals in
each metrics config, data is put on the queue with all information
needed to send to service"""
driverNiceName = metricSet[0]['source']['driver']['name']
if 'driverCounter' not in self.drivers[driverNiceName]:
self.drivers[driverNiceName]['driverCounter'] = self.drivers[
driverNiceName]['driverInterval']
else:
self.drivers[driverNiceName]['driverCounter'] += self.drivers[
driverNiceName]['driverInterval']
for metric in metricSet:
count = self.drivers[driverNiceName]['driverCounter']
metricInterval = int(metric['interval'])
if count % metricInterval == 0:
metricId = metric['id']
value = self.drivers[driverNiceName]['driver'].getData(metric)
dt = datetime.utcnow()
self.queue.enqueue(
self.sender.send_metric, metricId, value, dt)
def gcd(self, nums):
"""Recursively computes Greatest Common Divisor for a list of
numbers, used to compute the base scheduler interval for a
given set of metric intervals"""
if len(nums) == 1:
return nums[0]
if len(nums) == 0:
return None
if len(nums) >= 2:
a = nums[-1:][0]
b = nums[-2:-1][0]
while b:
a, b = b, a % b
nums = nums[:-2]
nums.append(a)
return self.gcd(nums)
def showConf(self):
"""Debug method to ensure config is loading correctly, and to
pretty print a config to clean up one with miffed formatting"""
return json.dumps(
self.config, sort_keys=True, indent=4, separators=(',', ': '))
class DakokuManager(object):
def __init__(self, config_path, schedule_path):
self.config_path = config_path
self.schedule_path = schedule_path
cfg = self._load_config()
try:
human_mode_min = cfg["human_mode"]
except:
human_mode_min = 0
try:
self.log_dir = cfg["log_dir"]
file_handler = logging.FileHandler(os.path.join(self.log_dir, "dakoku.log"), 'a+')
file_handler.level = LEVEL
log.addHandler(file_handler)
log.info("saving log to %s", self.log_dir)
except:
self.log_dir = None
sched = self._load_schedule()
start_date = dt.datetime.strptime(sched["valid"]["start"], '%Y-%m-%d').replace(tzinfo=pytz.timezone('Asia/Tokyo'))
end_date = dt.datetime.strptime(sched["valid"]["end"], '%Y-%m-%d').replace(tzinfo=pytz.timezone('Asia/Tokyo'))
log.info("dakoku is valid for %s - %s", start_date, end_date)
holidays = []
if "api_key" in cfg:
try:
holidays = self._get_holidays(cfg["api_key"], start_date, end_date)
log.info("holidays are skipped: %s", holidays)
except Exception as e:
log.error("failed to import holidays: %s", e)
else:
log.warn("No api_key specified. No holiday is registered!")
self.worker = DakokuWorker(cfg["host"], cfg["user"], cfg["pass"], holidays, self.log_dir)
self.register(sched["working"], start_date, end_date, holidays, human_mode_min)
def _load_config(self):
log.debug("loading config from %s", self.config_path)
with open(self.config_path, 'r') as f:
cfg = json.load(f)
return cfg
def _load_schedule(self):
log.debug("loading schedule from %s", self.schedule_path)
with open(self.schedule_path, 'r') as f:
cfg = json.load(f)
return cfg
def _get_holidays(self, api_key, start_date, end_date):
srv = Google(serviceName='calendar',
version='v3',
developerKey=api_key)
calendar_id = '*****@*****.**'
cal = srv.events().list(calendarId=calendar_id,
maxResults=30,
orderBy="startTime",
singleEvents=True,
timeMin=start_date.strftime('%Y-%m-%dT%H:%M:%SZ'),
timeMax=end_date.strftime('%Y-%m-%dT%H:%M:%SZ')).execute()
return [dt.datetime.strptime(e["start"]["date"], '%Y-%m-%d').replace(tzinfo=pytz.timezone("Asia/Tokyo")) for e in cal["items"]]
def register(self, working, start_date, end_date, holidays, human_mode_min=0):
self.scheduler = Scheduler(timezone=pytz.timezone('Asia/Tokyo'), logger=log)
today = dt.date.today()
for w in working:
# schedule shukkin
h, m = map(int, w["from"].split(':'))
fromtime = dt.time(h,m,tzinfo=pytz.timezone('Asia/Tokyo'))
d = dt.datetime.combine(today, fromtime) - dt.timedelta(minutes=human_mode_min)
trigger = CronTrigger(day_of_week=w["dayOfWeek"],
hour=d.hour, minute=d.minute,
start_date=start_date,
end_date=end_date,
timezone=pytz.timezone('Asia/Tokyo'))
self.scheduler.add_job(dispatch_after(human_mode_min * 60,
self.worker.work_start),
trigger)
# schedule taikin
h, m = map(int, w["till"].split(':'))
tilltime = dt.time(h,m,tzinfo=pytz.timezone('Asia/Tokyo'))
trigger = CronTrigger(day_of_week=w["dayOfWeek"],
hour=tilltime.hour, minute=tilltime.minute,
start_date=start_date,
end_date=end_date,
timezone=pytz.timezone('Asia/Tokyo'))
self.scheduler.add_job(dispatch_after(human_mode_min * 60,
self.worker.work_end),
trigger)
self.scheduler.print_jobs()
def start(self):
self.scheduler.start()
def shutdown(self):
self.scheduler.shutdown()
from apscheduler.schedulers.background import BackgroundScheduler
from django_apscheduler.jobstores import DjangoJobStore
from stock.crawler import twse
from apscheduler.executors.pool import ThreadPoolExecutor
from apscheduler.schedulers.blocking import BlockingScheduler
from datetime import datetime, timedelta
scheduler = BlockingScheduler()
scheduler.add_jobstore(DjangoJobStore())
# @scheduler.scheduled_job('cron', name='twse', second='*/3')
# def twse_job():
# # twse.spider_twse(datetime.today().strftime('%Y%m%d'), datetime.today().strftime('%Y%m%d')).crawler()
# print 'my_job is running, Now is %s' % datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# scheduler.print_jobs()
scheduler.start()
scheduler.print_jobs()
# job = scheduler.add_job(twse.spider_twse('20170620', '20170620').crawler, 'interval', minutes=5)
# scheduler = BlockingScheduler()
# scheduler.add_jobstore(DjangoJobStore())
# # scheduler.add_executor(ThreadPoolExecutor(10))
# def my_job():
# print 'my_job is running, Now is %s' % datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# scheduler.add_job(my_job, 'cron', second='*/5') # twse.spider_twse('20170621', '20170621').crawler
# scheduler.start()
# scheduler.print_jobs()
# print scheduler.get_jobs()
f"Trader For <{exchange_to_follow['Exchange']}> scheduled to run at:"
f"\n[Day Weeks: {day_of_week}]\t[Exchange closing hour: "
f"{exchange_closing.hour}:{exchange_closing.minute:02d} UTC]\t[Job starts at: "
f"{hour_minute.hour}:{hour_minute.minute:02d} UTC]\n")
#graps etoro instance with logged in driver and swticed to virtual/real account
etoro_instance = get_etoro_instance()
#scheduler = BlockingScheduler({'apscheduler.timezone': 'Europe/London'})
scheduler = BlockingScheduler({'apscheduler.timezone': 'UTC'})
# schedular to buy_trade
scheduler.add_job(buy_trade,
'cron',
args=[etoro_instance],
day_of_week=day_of_week,
minute=hour_minute.minute,
hour=hour_minute.hour)
# schedular to sell_trade
scheduler.add_job(sell_trade,
'interval',
args=[etoro_instance],
minutes=35)
# scheduler.add_job(buy_trade, 'cron', args=[etoro_instance],
# minute=9, hour=00)
#scheduler to display scheduled jobs
scheduler.add_job(lambda: scheduler.print_jobs(), 'interval', minutes=10)
scheduler.start()
class DakokuManager(object):
def __init__(self, config_path, schedule_path):
self.config_path = config_path
self.schedule_path = schedule_path
cfg = self._load_config()
try:
human_mode_min = cfg["human_mode"]
except:
human_mode_min = 0
try:
self.log_dir = cfg["log_dir"]
file_handler = logging.FileHandler(os.path.join(self.log_dir, "dakoku.log"), 'a+')
file_handler.level = LEVEL
log.addHandler(file_handler)
log.info("saving log to %s", self.log_dir)
except:
self.log_dir = None
self.api_key = cfg["api_key"]
sched = self._load_schedule()
start_date = dt.datetime.strptime(sched["valid"]["start"], '%Y-%m-%d').replace(tzinfo=pytz.timezone('Asia/Tokyo'))
end_date = dt.datetime.strptime(sched["valid"]["end"], '%Y-%m-%d').replace(tzinfo=pytz.timezone('Asia/Tokyo'))
log.info("dakoku is valid for %s - %s", start_date, end_date)
holidays = self._get_holidays(start_date, end_date)
self.worker = DakokuWorker(cfg["host"], cfg["user"], cfg["pass"], holidays, self.log_dir)
self.register(sched["working"], start_date, end_date, holidays, human_mode_min)
def _load_config(self):
log.debug("loading config from %s", self.config_path)
with open(self.config_path, 'r') as f:
cfg = json.load(f)
return cfg
def _load_schedule(self):
log.debug("loading schedule from %s", self.schedule_path)
with open(self.schedule_path, 'r') as f:
cfg = json.load(f)
return cfg
def _get_holidays(self, start_date, end_date):
holidays = japanese_holiday.getholidays(
str(self.api_key), # this function does not allow unicode
japanese_holiday.HOLIDAY_TYPE_OFFICIAL_JA,
start_date.strftime('%Y-%m-%d'),
end_date.strftime('%Y-%m-%d'),
)
ret = []
for hd in holidays:
date_str = hd['start']['date']
ret.append(dt.datetime.strptime(date_str, '%Y-%m-%d').replace(tzinfo=pytz.timezone('Asia/Tokyo')))
return ret
def register(self, working, start_date, end_date, holidays, human_mode_min=0):
self.scheduler = Scheduler(timezone=pytz.timezone('Asia/Tokyo'), logger=log)
today = dt.date.today()
for w in working:
# schedule shukkin
h, m = map(int, w["from"].split(':'))
fromtime = dt.time(h,m,tzinfo=pytz.timezone('Asia/Tokyo'))
d = dt.datetime.combine(today, fromtime) - dt.timedelta(minutes=human_mode_min)
trigger = CronTrigger(day_of_week=w["dayOfWeek"],
hour=d.hour, minute=d.minute,
start_date=start_date,
end_date=end_date,
timezone=pytz.timezone('Asia/Tokyo'))
self.scheduler.add_job(dispatch_after(human_mode_min * 60,
self.worker.work_start),
trigger)
# schedule taikin
h, m = map(int, w["till"].split(':'))
tilltime = dt.time(h,m,tzinfo=pytz.timezone('Asia/Tokyo'))
trigger = CronTrigger(day_of_week=w["dayOfWeek"],
hour=tilltime.hour, minute=tilltime.minute,
start_date=start_date,
end_date=end_date,
timezone=pytz.timezone('Asia/Tokyo'))
self.scheduler.add_job(dispatch_after(human_mode_min * 60,
self.worker.work_end),
trigger)
self.scheduler.print_jobs()
def start(self):
self.scheduler.start()
def shutdown(self):
self.scheduler.shutdown()
class MyScheduler:
def __init__(self):
self._scheduler = BlockingScheduler()
self._hue = HueWrapper()
self._enviro = EnviroWrapper()
self._kasa = KasaWrapper()
self._bright = 0
self._init()
self._jobs_list = [
self._manage_heater,
self._manage_lights,
]
self._jobs_cycle = itertools.cycle(self._jobs_list)
self.heater_on_for = 0
self.heater_off_for = 0
self.switch_on = False
def _init(self):
logging.basicConfig(level=logging.INFO)
logging.getLogger("apscheduler.scheduler").addFilter(MuteFilter())
self._hue.connect()
self._scheduler.add_job(func=self._get_sunset_sunrise, trigger=at_midnight)
self._scheduler.add_job(func=self._get_sunset_sunrise)
self._scheduler.add_job(self._manage_next, trigger=every_thirty_seconds)
self._scheduler.add_job(self._manage_all)
def start(self):
self._scheduler.print_jobs()
self._scheduler.start()
def stop(self):
self._scheduler.shutdown()
# if self._hue:
# self._hue.off()
def _manage_all(self):
for job in self._jobs_list:
job()
time.sleep(5)
def _manage_next(self):
job = next(self._jobs_cycle)
job()
def _manage_lights(self):
if self._hue.is_on:
self._hue.do_whatever()
def _manage_heater(self):
now = datetime.datetime.now()
weekday = now.weekday()
hour = now.hour
mins = now.minute
month = now.month
monday = 0
friday = 4
on_holiday = False
in_work_hours = not on_holiday \
and monday <= weekday <= friday \
and 8 <= hour <= 16 \
and (hour != 16 or mins <= 30)
is_spring = 3 <= month <= 5
is_summer = 6 <= month <= 8
is_autumn = 9 <= month <= 11
is_winter = month == 12 or month <= 2
is_morning = 0 <= hour <= 12
# is_early_morning = 0 <= hour <= 8
logging.info('weekday: {} hour: {} in_work_hours: {}'.format(weekday, hour, in_work_hours))
temperature = self._enviro.get_temperature()
logging.info('temperature: {}'.format(temperature))
if is_spring:
target_temperature = 18.0
elif is_summer:
target_temperature = 10.0
elif is_autumn:
target_temperature = 18.0
elif is_winter:
target_temperature = 18.0
else:
target_temperature = 18.0
if is_winter:
# if is_early_morning:
# target_temperature += 1
if is_morning:
target_temperature += 1
is_on = self._kasa.is_on()
logging.info('is_on: {0}'.format(is_on))
if is_on:
self.heater_on_for += 1
self.heater_off_for = 0
else:
self.heater_on_for = 0
self.heater_off_for += 1
logging.info('heater_on_for: {0}'.format(self.heater_on_for))
logging.info('heater_off_for: {0}'.format(self.heater_off_for))
cooler_thx = temperature > target_temperature or self.heater_on_for > 5
warmer_plz = in_work_hours and temperature < target_temperature - 2
if cooler_thx:
logging.info('cooler_thx')
self.switch_on = False
elif warmer_plz:
logging.info('warmer_plz')
if self.heater_off_for > 1:
logging.info('Duty cycle on')
self.switch_on = True
logging.info('switch_on: {0}'.format(self.switch_on))
if self.switch_on:
logging.info('Switching heater on')
self._kasa.switch_on()
elif cooler_thx:
logging.info('Switching heater off')
self._kasa.switch_off()
def _get_sunset_sunrise(self):
a = Astral()
leeds = a['Leeds']
today = datetime.date.today()
self._today_sun_data = leeds.sun(date=today, local=True)
self.timezone = leeds.timezone
logging.info(pprint.pformat(self._today_sun_data))
self.dawn = self._today_sun_data['dawn']
self.sunrise = self._today_sun_data['sunrise']
self.sunset = self._today_sun_data['sunset']
self.dusk = self._today_sun_data['dusk']
at_dawn = _get_cron_trigger_for_datetime(self.dawn)
at_sunrise = _get_cron_trigger_for_datetime(self.sunrise)
at_sunset = _get_cron_trigger_for_datetime(self.sunset)
at_dusk = _get_cron_trigger_for_datetime(self.dusk)
during_sunrise = IntervalTrigger(seconds=5, start_date=self.dawn, end_date=self.sunrise)
during_sunset = IntervalTrigger(seconds=5, start_date=self.sunset, end_date=self.dusk)
self._scheduler.add_job(func=self._at_dawn, trigger=at_dawn)
self._scheduler.add_job(func=self._during_sunrise, trigger=during_sunrise)
self._scheduler.add_job(func=self._at_sunrise, trigger=at_sunrise)
self._scheduler.add_job(func=self._at_sunset, trigger=at_sunset)
self._scheduler.add_job(func=self._during_sunset, trigger=during_sunset)
self._scheduler.add_job(func=self._at_dusk, trigger=at_dusk)
now = datetime.datetime.now(tz)
if now <= self.dawn:
day_factor = 0.0
elif self.dawn < now <= self.sunrise:
day_factor = colour_helper.get_day_factor(self.dawn, now, self.sunrise, True)
elif self.sunrise < now <= self.sunset:
day_factor = 1.0
elif self.sunset < now <= self.dusk:
day_factor = colour_helper.get_day_factor(self.sunset, now, self.dusk, False)
elif now < self.dusk:
day_factor = 0.0
else:
day_factor = 0.25
self._set_day_factor(day_factor)
def _at_dawn(self):
day_factor = 0.0
self._set_day_factor(day_factor)
logging.info('dawn')
def _at_sunrise(self):
day_factor = 1.0
self._set_day_factor(day_factor)
logging.info('sunrise')
def _at_sunset(self):
day_factor = 1.0
self._set_day_factor(day_factor)
logging.info('sunset')
def _at_dusk(self):
day_factor = 0.0
self._set_day_factor(day_factor)
logging.info('dusk')
def _during_sunrise(self):
day_factor = colour_helper.get_day_factor(self.dawn, datetime.datetime.now(tz), self.sunrise, True)
self._set_day_factor(day_factor)
def _during_sunset(self):
day_factor = colour_helper.get_day_factor(self.sunset, datetime.datetime.now(tz), self.dusk, False)
self._set_day_factor(day_factor)
def _set_day_factor(self, day_factor):
logging.info('day factor: {}'.format(day_factor))
colour_helper.set_day_factor(day_factor)
class JobManage():
def __init__(self):
jobstores = {'default': MemoryJobStore()}
executors = {
'default': ThreadPoolExecutor(50)
# 'processpool': ProcessPoolExecutor(3)
}
job_defaults = {'coalesce': False, 'max_instances': 50}
self.sched = BlockingScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults)
self.addError()
self.addJobExecuted()
def addJob(self, func, jobId=None, cron=None, args=[], kwargs={}):
'''
只支持cron的形式
* * * * * command
分 时 日 月 周 命令
第1列表示分钟1～59 每分钟用*或者 */1表示
第2列表示小时1～23（0表示0点）
第3列表示日期1～31
第4列表示月份1～12
第5列标识号星期0～6（0表示星期天）
第6列要运行的命令
'''
if cron is None:
raise Exception("cron cannot be Null")
(minute, hour, day, month, week) = cron.split(" ")
self.sched.add_job(func,
trigger='cron',
id=jobId,
hour=hour,
minute=minute,
day=day,
month=month,
week=week,
args=args,
kwargs=kwargs)
def removeJob(self, jobId):
self.sched.remove_job(jobId)
def start(self):
self.sched.start()
def shutdown(self):
self.sched.shutdown()
def printJobs(self):
self.sched.print_jobs()
def getJobs(self):
return self.sched.get_jobs()
def addError(self, func=None):
if func is None:
func = self.listener
self.sched.add_listener(func, EVENT_JOB_ERROR)
def addJobExecuted(self, func=None):
if func is None:
func = self.listener
self.sched.add_listener(func, EVENT_JOB_EXECUTED)
def listener(self, event):
if event.exception:
log.error("任务【%s】 任务出错 : %s" % (event.job_id, event.traceback))
else:
log.debug("任务【%s】已经跑完，结束时间 : %s " % (event.job_id, getNow()))
# jobMange = JobManage()
import tushare as ts
from apscheduler.schedulers.blocking import BlockingScheduler
class StockDataService():
def __init__(self):
return
def get_cash_flow(self):
ts.get_cash_flow()
return
def my_job(self):
print('hello world')
def main():
a = StockDataService()
sched.print_jobs()
sched.add_job(a.my_job, 'interval', seconds=5)
sched.start()
sched = BlockingScheduler()
main()
sched.print_jobs()
from django_apscheduler.jobstores import DjangoJobStore
from stock.crawler import twse
from apscheduler.executors.pool import ThreadPoolExecutor
from apscheduler.schedulers.blocking import BlockingScheduler
from datetime import datetime,timedelta
scheduler = BlockingScheduler()
scheduler.add_jobstore(DjangoJobStore())
# @scheduler.scheduled_job('cron', name='twse', second='*/3')
# def twse_job():
# # twse.spider_twse(datetime.today().strftime('%Y%m%d'), datetime.today().strftime('%Y%m%d')).crawler()
# print 'my_job is running, Now is %s' % datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# scheduler.print_jobs()
scheduler.start()
scheduler.print_jobs()
# job = scheduler.add_job(twse.spider_twse('20170620', '20170620').crawler, 'interval', minutes=5)
# scheduler = BlockingScheduler()
# scheduler.add_jobstore(DjangoJobStore())
# # scheduler.add_executor(ThreadPoolExecutor(10))
# def my_job():
# print 'my_job is running, Now is %s' % datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# scheduler.add_job(my_job, 'cron', second='*/5') # twse.spider_twse('20170621', '20170621').crawler
# scheduler.start()
# scheduler.print_jobs()
# print scheduler.get_jobs()
class DakokuManager(object):
def __init__(self, config_path, schedule_path):
self.config_path = config_path
self.schedule_path = schedule_path
cfg = self._load_config()
try:
human_mode_min = cfg["human_mode"]
except:
human_mode_min = 0
try:
self.log_dir = cfg["log_dir"]
file_handler = logging.FileHandler(
os.path.join(self.log_dir, "dakoku.log"), 'a+')
file_handler.level = LEVEL
log.addHandler(file_handler)
log.info("saving log to %s", self.log_dir)
except:
self.log_dir = None
sched = self._load_schedule()
start_date = dt.datetime.strptime(
sched["valid"]["start"],
'%Y-%m-%d').replace(tzinfo=pytz.timezone('Asia/Tokyo'))
end_date = dt.datetime.strptime(
sched["valid"]["end"],
'%Y-%m-%d').replace(tzinfo=pytz.timezone('Asia/Tokyo'))
log.info("dakoku is valid for %s - %s", start_date, end_date)
holidays = []
if "api_key" in cfg:
try:
holidays = self._get_holidays(cfg["api_key"], start_date,
end_date)
log.info("holidays are skipped: %s", holidays)
except Exception as e:
log.error("failed to import holidays: %s", e)
else:
log.warn("No api_key specified. No holiday is registered!")
self.worker = DakokuWorker(cfg["host"], cfg["user"], cfg["pass"],
holidays, self.log_dir)
self.register(sched["working"], start_date, end_date, holidays,
human_mode_min)
def _load_config(self):
log.debug("loading config from %s", self.config_path)
with open(self.config_path, 'r') as f:
cfg = json.load(f)
return cfg
def _load_schedule(self):
log.debug("loading schedule from %s", self.schedule_path)
with open(self.schedule_path, 'r') as f:
cfg = json.load(f)
return cfg
def _get_holidays(self, api_key, start_date, end_date):
srv = Google(serviceName='calendar',
version='v3',
developerKey=api_key)
calendar_id = '*****@*****.**'
cal = srv.events().list(
calendarId=calendar_id,
maxResults=30,
orderBy="startTime",
singleEvents=True,
timeMin=start_date.strftime('%Y-%m-%dT%H:%M:%SZ'),
timeMax=end_date.strftime('%Y-%m-%dT%H:%M:%SZ')).execute()
return [
dt.datetime.strptime(
e["start"]["date"],
'%Y-%m-%d').replace(tzinfo=pytz.timezone("Asia/Tokyo"))
for e in cal["items"]
]
def register(self,
working,
start_date,
end_date,
holidays,
human_mode_min=0):
self.scheduler = Scheduler(timezone=pytz.timezone('Asia/Tokyo'),
logger=log)
today = dt.date.today()
for w in working:
# schedule shukkin
h, m = map(int, w["from"].split(':'))
fromtime = dt.time(h, m, tzinfo=pytz.timezone('Asia/Tokyo'))
d = dt.datetime.combine(
today, fromtime) - dt.timedelta(minutes=human_mode_min)
trigger = CronTrigger(day_of_week=w["dayOfWeek"],
hour=d.hour,
minute=d.minute,
start_date=start_date,
end_date=end_date,
timezone=pytz.timezone('Asia/Tokyo'))
self.scheduler.add_job(
dispatch_after(human_mode_min * 60, self.worker.work_start),
trigger)
# schedule taikin
h, m = map(int, w["till"].split(':'))
tilltime = dt.time(h, m, tzinfo=pytz.timezone('Asia/Tokyo'))
trigger = CronTrigger(day_of_week=w["dayOfWeek"],
hour=tilltime.hour,
minute=tilltime.minute,
start_date=start_date,
end_date=end_date,
timezone=pytz.timezone('Asia/Tokyo'))
self.scheduler.add_job(
dispatch_after(human_mode_min * 60, self.worker.work_end),
trigger)
self.scheduler.print_jobs()
def start(self):
self.scheduler.start()
def shutdown(self):
self.scheduler.shutdown()
from apscheduler.schedulers.blocking import BlockingScheduler scheduler = BlockingScheduler(daemon=True) scheduler.add_job(lambda: scheduler.print_jobs(), 'interval', seconds=2) scheduler.start()
# Schedule/Reschedule twitter streaming for all remaining games
remaining_games = schedule.remaining_games()
for game_date in remaining_games.date.unique():
games = remaining_games[remaining_games.date == game_date]
now = datetime.now(eastern)
start_time = games.est.min().to_datetime() - timedelta(minutes=15)
end_time = games.est.max().to_datetime() + timedelta(hours=3, minutes=15)
# Logic to check if a job should be running while the application is starting
if start_time < now < end_time: # Games happening now
start_time = now
elif now > end_time: # Games are over, but still the date of the games (rare)
continue
runtime = (end_time - start_time).seconds
sched.add_job(q.enqueue, args=(twitter.run, runtime), trigger='date', run_date=start_time,
id=str(games.date) + ':producer', misfire_grace_time=30)
sched.add_job(q.enqueue, args=(consumer.run, runtime), trigger='date', run_date=start_time,
id=str(games.date) + ':consumer', misfire_grace_time=30)
sched.add_job(q.enqueue, args=(play_by_play.run, runtime), trigger='date', run_date=start_time,
id=str(games.date) + ':pbp', misfire_grace_time=30)
sched.add_job(q.enqueue, args=(models.run, start_time), trigger='date', run_date=end_time,
id=str(games.date) + ':addtodb', misfire_grace_time=30)
if __name__ == '__main__':
sched.print_jobs()
sched.start()
