def run(self, force_current_time="no", force_last_time="no"):
"""
Runs all registered jobs that currently need to perform
updates. If the optional *force_current_time* or
*force_last_time* arguments are included, jobs will be run as if
working at those times. (This is useful for testing or force
loading old data.)
"""
jobs = list(self._jobs)
random.shuffle(jobs)
for job_name in jobs:
if force_current_time == "no" or force_current_time == None:
start_time = make_datetime(datetime.utcnow())
else:
start_time = make_datetime(force_current_time)
(period, func) = self._jobs[job_name]
current_time = bin_datetime(period, start_time)
if force_last_time == "no" or force_last_time == None:
last_time = self._last_run_times.get(job_name, None)
else:
last_time = make_datetime(force_last_time)
if last_time is None or last_time < current_time:
try:
log.debug("%s %s %s %s %s" % (job_name, period, last_time, current_time, func))
func(last_time, current_time)
self._last_run_times[job_name] = current_time
except:
# We don't fail on errors. The func has to do something
# if it wants to be special.
log.exception("Job manager: error running %s" % job_name)
pass
def test_bin_datetime_7_days_alt_z(self):
"bin_datetime(timedelta(days=1), ..., z=DT_EPOCH+timedelta(days=1))"
self.assertEqual(
bin_datetime(timedelta(days=7),
self.dt_20100203T040506_007008,
z=netsa.data.times.DT_EPOCH+timedelta(days=1)),
self.dt_20100129T000000)
def nice_second_ticks(lo, hi, ticks=5, inside=False):
lo_sec = bin_datetime(timedelta(seconds=1), lo)
hi_sec = bin_datetime(timedelta(seconds=1), hi)
if hi - hi_sec:
hi_sec += timedelta(seconds=1)
delta_secs = (hi_sec - lo_sec).seconds + (hi_sec - lo_sec).days * 86400
if delta_secs < (ticks - 1):
raise ValueError()
t_min, t_max, t_iter = nice_ticks(0, delta_secs, ticks, inside,
int_60_intervals, base=60.0)
min_sec = lo_sec + timedelta(seconds=int(t_min))
max_sec = lo_sec + timedelta(seconds=int(t_max))
def s_iter():
for t in t_iter:
yield lo_sec + timedelta(seconds=int(t))
return (min_sec, max_sec, s_iter())
def nice_minute_ticks(lo, hi, ticks=5, inside=False):
lo_min = bin_datetime(timedelta(minutes=1), lo)
hi_min = bin_datetime(timedelta(minutes=1), hi)
if hi - hi_min:
hi_min += timedelta(minutes=1)
delta_mins = (hi_min - lo_min).seconds / 60 + (hi_min - lo_min).days * 1440
if delta_mins < (ticks - 1):
raise ValueError()
t_min, t_max, t_iter = nice_ticks(0, delta_mins, ticks, inside,
intervals=int_60_intervals, base=60.0)
min_min = lo_min + timedelta(minutes=int(t_min))
min_max = lo_min + timedelta(minutes=int(t_max))
def m_iter():
for t in t_iter:
yield lo_min + timedelta(minutes=int(t))
return (min_min, min_max, m_iter())
def nice_day_ticks(lo, hi, ticks=5, inside=False):
lo_day = bin_datetime(timedelta(days=1), lo)
hi_day = bin_datetime(timedelta(days=1), hi)
if hi - hi_day:
hi_day += timedelta(days=1)
delta_days = (hi_day - lo_day).days
if delta_days < (ticks - 1):
raise ValueError()
t_min, t_max, t_iter = nice_ticks(0, delta_days, ticks, inside,
int_intervals)
day_min = lo_day + timedelta(days=int(t_min))
day_max = lo_day + timedelta(days=int(t_max))
def d_iter():
for t in t_iter:
yield lo_day + timedelta(days=int(t))
return (day_min, day_max, d_iter())
def nice_week_ticks(lo, hi, ticks=5, inside=False):
lo_week = bin_datetime(timedelta(days=7), lo, WEEK_EPOCH)
hi_week = bin_datetime(timedelta(days=7), hi, WEEK_EPOCH)
if hi - hi_week:
hi_week += timedelta(days=7)
delta_weeks = (hi_week - lo_week).days / 7
if delta_weeks < (ticks - 1):
raise ValueError()
t_min, t_max, t_iter = nice_ticks(0, delta_weeks, ticks, inside,
int_intervals)
week_min = lo_week + timedelta(days=int(7*t_min))
week_max = lo_week + timedelta(days=int(7*t_max))
def w_iter():
for t in t_iter:
yield lo_week + timedelta(days=int(7*t))
return (week_min, week_max, w_iter())
def nice_hour_ticks(lo, hi, ticks=5, inside=False):
lo_hour = bin_datetime(timedelta(hours=1), lo)
hi_hour = bin_datetime(timedelta(hours=1), hi)
if hi - hi_hour:
hi_hour += timedelta(hours=1)
delta_hours = ((hi_hour - lo_hour).seconds / 3600 +
(hi_hour - lo_hour).days * 24)
if delta_hours < (ticks - 1):
raise ValueError()
t_min, t_max, t_iter = nice_ticks(0, delta_hours, ticks, inside,
intervals=int_12_intervals, base=24.0)
hour_min = lo_hour + timedelta(hours=int(t_min))
hour_max = lo_hour + timedelta(hours=int(t_max))
def h_iter():
for t in t_iter:
yield lo_hour + timedelta(hours=int(t))
return (hour_min, hour_max, h_iter())
def epoch(self):
"""
A :class:`datetime <datetime.datetime>` instance representing
the epoch for aligning various time bins such as intervals and
spans. This is primarily useful for aligning these bins to a
particular day of the week. By default, bins align to Monday
within any given week.
"""
e = self._epoch or self.default_epoch
return times.bin_datetime(self.interval, e)
def nice_arb_ticks(lo, hi, ticks=5, inside=False):
base = bin_datetime(timedelta(minutes=1), lo)
lo_sec = (lo - base).seconds + (lo - base).days * 86400
hi_sec = (hi - base).seconds + (hi - base).days * 86400
t_min, t_max, t_iter = nice_ticks(lo_sec, hi_sec, ticks, inside)
min_sec = base + timedelta(seconds=t_min)
max_sec = base + timedelta(seconds=t_max)
def s_iter():
for t in t_iter:
yield base + timedelta(seconds=t)
return (min_sec, max_sec, s_iter())
def _date_bin(self, date):
return times.bin_datetime(self.interval, date, self.epoch)
def test_bin_datetime_year(self):
"bin_datetime('year', ...)",
self.assertEqual(
bin_datetime("year",
self.dt_20100203T040506_007008),
self.dt_20100101T000000)
def test_bin_datetime_month(self):
"bin_datetime('month', ...)",
self.assertEqual(
bin_datetime("month",
self.dt_20100203T040506_007008),
self.dt_20100201T000000)
def test_bin_datetime_7_days(self):
"bin_datetime(timedelta(days=1), ...)"
self.assertEqual(
bin_datetime(timedelta(days=7),
self.dt_20100203T040506_007008),
self.dt_20100128T000000)
def test_bin_datetime_3_hours(self):
"bin_datetime(timedelta(hours=3), ...)"
self.assertEqual(
bin_datetime(timedelta(hours=3),
self.dt_20100203T040506_007008),
self.dt_20100203T030000)
def test_bin_datetime_30_minutes(self):
"bin_datetime(timedelta(minutes=30), ...)"
self.assertEqual(
bin_datetime(timedelta(minutes=30),
self.dt_20100203T040506_007008),
self.dt_20100203T040000)
def test_bin_datetime_5_seconds(self):
"bin_datetime(timedelta(seconds=5), ...)"
self.assertEqual(
bin_datetime(timedelta(seconds=5),
self.dt_20100203T040506_007008),
self.dt_20100203T040505)
