def run(self, force_current_time="no", force_last_time="no"):
"""
Runs all registered jobs that currently need to perform
updates. If the optional *force_current_time* or
*force_last_time* arguments are included, jobs will be run as if
working at those times. (This is useful for testing or force
loading old data.)
"""
jobs = list(self._jobs)
random.shuffle(jobs)
for job_name in jobs:
if force_current_time == "no" or force_current_time == None:
start_time = make_datetime(datetime.utcnow())
else:
start_time = make_datetime(force_current_time)
(period, func) = self._jobs[job_name]
current_time = bin_datetime(period, start_time)
if force_last_time == "no" or force_last_time == None:
last_time = self._last_run_times.get(job_name, None)
else:
last_time = make_datetime(force_last_time)
if last_time is None or last_time < current_time:
try:
log.debug("%s %s %s %s %s" % (job_name, period, last_time, current_time, func))
func(last_time, current_time)
self._last_run_times[job_name] = current_time
except:
# We don't fail on errors. The func has to do something
# if it wants to be special.
log.exception("Job manager: error running %s" % job_name)
pass
def nice_month_ticks(lo, hi, ticks=5, inside=False):
lo_year = lo.year
lo_month = lo.month - 1
hi_year = hi.year
hi_month = hi.month - 1
if hi - make_datetime(datetime(hi_year, hi.month, 1)):
if hi_month == 11:
hi_month = 0
hi_year += 1
else:
hi_month += 1
delta_year = hi_year - lo_year
if delta_year * 12 + hi_month - lo_month < (ticks - 1):
raise ValueError()
t_min, t_max, t_iter = nice_ticks(lo_month, delta_year * 12 + hi_month,
ticks, inside,
intervals=int_12_intervals, base=12.0)
t_min_year = int(lo_year + t_min / 12)
t_min_month = int(t_min % 12) + 1
t_max_year = int(lo_year + t_max / 12)
t_max_month = int(t_max % 12) + 1
month_min = make_datetime(datetime(t_min_year, t_min_month, 1))
month_max = make_datetime(datetime(t_max_year, t_max_month, 1))
def m_iter():
for t in t_iter:
year = int(lo_year + t / 12)
month = int(t % 12) + 1
yield make_datetime(datetime(year, month, 1))
return (month_min, month_max, m_iter())
def test_nice_time_ticks_16(self):
(a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
make_datetime('2013-01-01'))
self.assertEqual(a, make_datetime('2011-01-01'))
self.assertEqual(b, make_datetime('2013-01-01'))
self.assertEqual(list(i), [make_datetime(x) for x in
['2011-01-01', '2011-07-01', '2012-01-01',
'2012-07-01', '2013-01-01']])
def test_nice_time_ticks_11(self):
(a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
make_datetime('2011-01-14'))
self.assertEqual(a, make_datetime('2011-01-01'))
self.assertEqual(b, make_datetime('2011-01-16'))
self.assertEqual(list(i), [make_datetime(x) for x in
['2011-01-01', '2011-01-04', '2011-01-07',
'2011-01-10', '2011-01-13', '2011-01-16']])
def test_nice_time_ticks_17(self):
(a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
make_datetime('2525-01-01'))
self.assertEqual(a, make_datetime('2000-01-01'))
self.assertEqual(b, make_datetime('2600-01-01'))
self.assertEqual(list(i), [make_datetime(x) for x in
['2000-01-01', '2100-01-01', '2200-01-01',
'2300-01-01', '2400-01-01', '2500-01-01',
'2600-01-01']])
def test_nice_time_ticks_12(self):
(a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
make_datetime('2011-02-24'))
self.assertEqual(a, make_datetime('2011-01-01'))
self.assertEqual(b, make_datetime('2011-03-02'))
self.assertEqual(list(i), [make_datetime(x) for x in
['2011-01-01', '2011-01-11', '2011-01-21',
'2011-01-31', '2011-02-10', '2011-02-20',
'2011-03-02']])
def test_nice_time_ticks_9(self):
(a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
make_datetime('2011-01-03'))
self.assertEqual(a, make_datetime('2011-01-01'))
self.assertEqual(b, make_datetime('2011-01-03'))
self.assertEqual(list(i), [make_datetime(x) for x in
['2011-01-01T00', '2011-01-01T12',
'2011-01-02T00', '2011-01-02T12',
'2011-01-03T00']])
def test_nice_time_ticks_15(self):
(a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
make_datetime('2011-12-01'), ticks=12)
self.assertEqual(a, make_datetime('2011-01-01'))
self.assertEqual(b, make_datetime('2011-12-01'))
self.assertEqual(list(i), [make_datetime(x) for x in
['2011-01-01', '2011-02-01', '2011-03-01',
'2011-04-01', '2011-05-01', '2011-06-01',
'2011-07-01', '2011-08-01', '2011-09-01',
'2011-10-01', '2011-11-01', '2011-12-01']])
def test_nice_time_ticks_6(self):
(a, b, i) = nice_time_ticks(make_datetime('2011-01-01T00:00:00.000'),
make_datetime('2011-01-01T01:00:00.000'))
self.assertEqual(a, make_datetime('2011-01-01T00:00:00.000'))
self.assertEqual(b, make_datetime('2011-01-01T01:00:00.000'))
self.assertEqual(list(i), [make_datetime(x) for x in
['2011-01-01T00:00:00.000',
'2011-01-01T00:15:00.000',
'2011-01-01T00:30:00.000',
'2011-01-01T00:45:00.000',
'2011-01-01T01:00:00.000']])
def test_nice_time_ticks_1(self):
(a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
make_datetime('2011-01-01'))
self.assertEqual(a, make_datetime('2010-12-31T23:59:59.500'))
self.assertEqual(b, make_datetime('2011-01-01T00:00:00.500'))
self.assertEqual(list(i), [make_datetime(x) for x in
['2010-12-31T23:59:59.500',
'2010-12-31T23:59:59.750',
'2011-01-01T00:00:00.000',
'2011-01-01T00:00:00.250',
'2011-01-01T00:00:00.500']])
def test_nice_time_ticks_7(self):
(a, b, i) = nice_time_ticks(make_datetime('2011-01-01T00:00:00.000'),
make_datetime('2011-01-01T14:00:00.000'))
self.assertEqual(a, make_datetime('2011-01-01T00:00:00.000'))
self.assertEqual(b, make_datetime('2011-01-01T15:00:00.000'))
self.assertEqual(list(i), [make_datetime(x) for x in
['2011-01-01T00:00:00.000',
'2011-01-01T03:00:00.000',
'2011-01-01T06:00:00.000',
'2011-01-01T09:00:00.000',
'2011-01-01T12:00:00.000',
'2011-01-01T15:00:00.000']])
def test_nice_time_ticks_5(self):
(a, b, i) = nice_time_ticks(make_datetime('2011-01-01T00:00:00.000'),
make_datetime('2011-01-01T00:10:00.000'))
self.assertEqual(a, make_datetime('2011-01-01T00:00:00.000'))
self.assertEqual(b, make_datetime('2011-01-01T00:10:00.000'))
self.assertEqual(list(i), [make_datetime(x) for x in
['2011-01-01T00:00:00.000',
'2011-01-01T00:02:00.000',
'2011-01-01T00:04:00.000',
'2011-01-01T00:06:00.000',
'2011-01-01T00:08:00.000',
'2011-01-01T00:10:00.000']])
def nice_year_ticks(lo, hi, ticks=5, inside=False):
lo_year = lo.year
hi_year = hi.year
if hi - make_datetime(datetime(hi_year, 1, 1)):
hi_year += 1
if hi_year - lo_year < (ticks - 1):
raise ValueError()
t_min, t_max, t_iter = nice_ticks(lo_year, hi_year, ticks, inside,
intervals=int_intervals)
year_min = make_datetime(datetime(int(t_min), 1, 1))
year_max = make_datetime(datetime(int(t_max), 1, 1))
def y_iter():
for t in t_iter:
yield make_datetime(datetime(int(t), 1, 1))
return (year_min, year_max, y_iter())
def parse_param_date_value(param, value):
try:
value = make_datetime(value)
except ValueError:
error = ParamError(param, value, "not a valid date/time")
raise error
return value
def parse_param_date_value(param, value, is_default):
if is_default:
default_msg = " in script default"
else:
default_msg = ""
try:
value = make_datetime(value)
except ValueError:
error = ParamError(param, value, "not a valid date/time" + default_msg)
raise error
return value
def _make_from_meta(cls, **kwargs):
mdata = kwargs.setdefault('metadata', {})
version = int(mdata['golem_version'])
if version > GOLEM_VERSION:
error = GolemScriptError(
"Unknown golem version %s" % repr(version))
raise error
kwargs['_version'] = version
if 'golem_span' in mdata:
kwargs['span'] = \
times.make_timedelta(mdata.pop('golem_span'))
if 'golem_interval' in mdata:
kwargs['interval'] = \
times.make_timedelta(mdata.pop('golem_interval'))
if 'golem_lag' in mdata:
kwargs['lag'] = \
times.make_timedelta(mdata.pop('golem_lag'))
if 'golem_epoch' in mdata:
kwargs['epoch'] = \
times.make_datetime(mdata.pop('golem_epoch'))
if 'golem_realtime' in mdata:
kwargs['realtime'] = bool(mdata.pop('golem_realtime'))
if 'golem_name' in mdata:
kwargs['name'] = mdata.pop('golem_name')
if 'golem_suite_name' in mdata:
kwargs['suite_name'] = mdata.pop('golem_suite_name')
if 'golem_inputs' in mdata:
inputs = []
for meta, spec in mdata.pop('golem_inputs'):
inputs.append((cls._make_from_meta(**meta), spec))
kwargs['golem_inputs'] = inputs
if 'golem_tags' in mdata:
kwargs['tags'] = mdata.pop('golem_tags')
if 'golem_loops' in mdata:
kwargs['loops'] = mdata.pop('golem_loops')
if 'golem_flow_maps' in mdata:
kwargs['flow_maps'] = mdata.pop('golem_flow_maps')
if 'golem_arg_tags' in mdata:
kwargs['arg_tags'] = mdata.pop('golem_arg_tags')
if 'golem_repository' in mdata:
kwargs['repository'] = mdata.pop('golem_repository')
if 'golem_output_templates' in mdata:
kwargs['output_templates'] = mdata.pop('golem_output_templates')
if 'golem_input_templates' in mdata:
kwargs['input_templates'] = mdata.pop('golem_input_templates')
if 'golem_query_templates' in mdata:
kwargs['query_templates'] = mdata.pop('golem_query_templates')
if 'output_groups' in mdata:
kwargs['output_groups'] = mdata.pop('output_groups')
if 'input_groups' in mdata:
kwargs['input_groups'] = mdata.pop('input_groups')
return cls(**kwargs)
def store(self, item):
"""
Stores the provided data structure into the associated file,
along with a timestamp representing the current time.
"""
try:
if DEBUG:
print >> sys.stderr, "save to sig file %s" % self.file
fh = open(self.file, 'wb')
pickle.dump((item, make_datetime(datetime.now())), fh)
fh.close()
except pickle.PicklingError, e:
msg = "problem storing %s : %s" % (self.file, e)
raise LedgerStoreError, msg
def date_from_file(file):
"""
Attempt to extract dates from a filename. The filename can be a
full pathname or relative path. Dates are presumed to exist
somewhere in the pathname. See :func:`split_on_date` for more
detail on how dates are parsed from filenames.
"""
(dir, chop) = split_on_date(file)
ymd = {}
for (i, field) in ( (1, 'year'), (3, 'month'), (5, 'day'),
(7, 'hour'), (9, 'minute'), (11, 'second') ):
if field in ('day', 'month'):
ymd[field] = chop[i] or 1
else:
ymd[field] = chop[i] or 0
try:
return make_datetime(datetime(**ymd), utc_only=True)
except ValueError, e:
msg = str(e)
msg += "\nproblem extracting date from %s" % file
raise DateFileParseError, msg
def __init__(self, span=None, interval=None,
name=None, suite_name=None, epoch=None,
lag=None, realtime=False, repository=None,
tags=None, arg_tags=None, loops=None,
flow_maps=None, golem_inputs=None,
output_templates=None, input_templates=None,
query_templates=None, input_groups=None,
output_groups=None, _version=None, **kwargs):
"""
Creates a new :class:`Golem` object from the various data that
make up a golem script definition. This should never need to be
used by anything outside of the :mod:`golem.model` module. See
instead :func:`golem.model.parse_golem_metadata` for a way to
retrieve golem script information from serialized metadata or
existing scripts.
"""
Script.__init__(self, **kwargs)
self._version = _version
self.name = name
"""
The short name of this golem script, if any.
"""
if self.name is None:
try:
self.name = os.path.basename(main.__file__)
self.name.replace('.py', '')
except AttributeError:
self.name = "python"
self.suite = suite_name
"""
The short name of this golem script's suite, if any.
"""
self._epoch = None
"""
The epoch which all time bins are relative to. Defaults to
midnight of the first Monday after the standard unix epoch.
"""
if epoch:
self._epoch = times.make_datetime(epoch)
self.interval = \
times.make_timedelta(interval or self.default_interval)
"""
A :class:`timedelta` representing the process interval
of this :class:`Golem` instance.
"""
self.span = times.make_timedelta(span or self.interval)
"""
A :class:`timedelta` representing the data window of
this :class:`Golem` instance.
"""
self.repository = repository
"""
The default path, if any, for the output results data
repository for this :class:`Golem` instance.
"""
self.realtime = realtime or False
"""
A boolean value indicating whether results for this
:class:`Golem` script can be generated in real time
or not. If true, forces a *lag* of 0.
"""
self._lag = None
if lag and not self.realtime:
self._lag = times.make_timedelta(lag)
if not isinstance(tags, dict):
tags = dict(tags or ())
self.tags = tags
if not isinstance(arg_tags, dict):
arg_tags = dict(arg_tags or ())
self.arg_tags = arg_tags
self.flow_maps = tuple(flow_maps or ())
for x in self.flow_maps:
name, params = x
if isinstance(x, tuple) and isinstance(params, dict):
continue
maps = []
for name, params in self.flow_maps:
maps.append((name, dict(params)))
self.flow_maps = tuple(flow_maps)
break
self.loops = tuple(loops or ())
for item in self.loops:
name, x = item
vals, groups, group_name, sep = x
if all(isinstance(y, tuple) for y in (item, x, vals)) and \
not isinstance(vals, basestring) and \
not (groups and not group_name) and \
(groups and isinstance(groups, dict)):
continue
loops = []
for name, (vals, groups, group_name, sep) in self.loops:
if isinstance(vals, basestring):
vals = (vals,)
if groups:
if not group_name:
group_name = name + '_group'
if not isinstance(groups, dict):
groups = dict(groups)
loops.append((name, (vals, groups, group_name, sep)))
self.loops = tuple(loops)
break
self.input_templates = tuple(input_templates or ())
for x in self.input_templates:
name, t, spec = x
if isinstance(x, tuple):
continue
templates = []
for name, t, spec in self.input_templates:
templates.append((name, t, spec))
self.input_templates = tuple(templates)
break
self.output_templates = tuple(output_templates or ())
for x in self.output_templates:
if isinstance(x, tuple):
continue
templates = []
for name, t, spec in self.output_templates:
templates.append((name, t, spec))
self.output_templates = tuple(templates)
break
self.query_templates = tuple(query_templates or ())
for x in self.query_templates:
if isinstance(x, tuple):
continue
templates = []
for name, t, spec in self.query_templates:
templates.append((name, t, spec))
self.query_templates = tuple(templates)
break
self.input_groups = tuple(input_groups or ())
self.output_groups = tuple(output_groups or ())
self.golem_inputs = tuple(golem_inputs or ())
self._validate_params()
def test_make_datetime_iso_usec_tzminute(self):
"make_datetime('2010-02-03T04:05:06.007008+09:10')"
self.assertEqual(
self.dt_20100203T040506_007008_0910,
make_datetime('2010-02-03T04:05:06.007008+09:10'))
def test_make_datetime_iso_usec(self):
"make_datetime('2010-02-03T04:05:06.007008')"
self.assertEqual(
self.dt_20100203T040506_007008,
make_datetime('2010-02-03T04:05:06.007008'))
def test_make_datetime_iso_second_space(self):
"make_datetime('2010-02-03 04:05:06')"
self.assertEqual(
self.dt_20100203T040506,
make_datetime('2010-02-03 04:05:06'))
def test_make_datetime_iso_minute(self):
"make_datetime('2010-02-03T04:05')"
self.assertEqual(
self.dt_20100203T040500,
make_datetime('2010-02-03T04:05'))
def test_make_datetime_iso_hour(self):
"make_datetime('2010-02-03T04')"
self.assertEqual(
self.dt_20100203T040000,
make_datetime('2010-02-03T04'))
def test_make_datetime_iso_day(self):
"make_datetime('2010-02-03')"
self.assertEqual(
self.dt_20100203,
make_datetime('2010-02-03'))
def test_make_datetime_long_20100203T040506(self):
"make_datetime(1265169906L)"
self.assertEqual(
self.dt_20100203T040506,
make_datetime(1265169906L))
def test_make_datetime_int_epoch(self):
"make_datetime(0)"
self.assertEqual(
netsa.data.times.DT_EPOCH,
make_datetime(0))
def test_make_datetime_silk_old_msec(self):
"make_datetime('02/03/2010:04:05:06.007')"
self.assertEqual(
self.dt_20100203T040506_007,
make_datetime('02/03/2010:04:05:06.007'))
def test_make_datetime_silk_old_second(self):
"make_datetime('02/03/2010:04:05:06')"
self.assertEqual(
self.dt_20100203T040506,
make_datetime('02/03/2010:04:05:06'))
def test_make_datetime_silk_old_minute(self):
"make_datetime('02/03/2010:04:05')"
self.assertEqual(
self.dt_20100203T040500,
make_datetime('02/03/2010:04:05'))