def test_special_range(self):
result_begin = datetime.datetime(2014, 8, 1, 0, 0, 0, 0)
result_end = datetime.datetime(2014, 8, 2, 0, 0, 0, 0)
exp = timex.parse("day($test_thingy)")
t = exp(test_thingy=self.dt)
self.assertTrue(t.is_range)
self.assertEqual(t.begin, result_begin)
self.assertEqual(t.end, result_end)
exp = timex.parse("day")
t = exp(timestamp=self.dt)
self.assertTrue(t.is_range)
self.assertEqual(t.begin, result_begin)
self.assertEqual(t.end, result_end)
def __init__(self, config):
self.included_types = []
self.excluded_types = []
if 'event_type' in config:
event_types = config['event_type']
if isinstance(event_types, six.string_types):
event_types = [event_types]
for t in event_types:
if t.startswith('!'):
self.excluded_types.append(t[1:])
else:
self.included_types.append(t)
else:
self.included_types.append('*')
if self.excluded_types and not self.included_types:
self.included_types.append('*')
if 'number' in config:
self.number = config['number']
else:
self.number = 1
if 'timestamp' in config:
self.timestamp = timex.parse(config['timestamp'])
else:
self.timestamp = None
self.map_distinguished_by = dict()
if 'map_distinguished_by' in config:
self.map_distinguished_by = config['map_distinguished_by']
self.traits = dict()
if 'traits' in config:
for trait, criterion in config['traits'].items():
self.traits[trait] = Criterion.get_from_expression(criterion,
trait)
def __init__(self, config):
self.included_types = []
self.excluded_types = []
if 'event_type' in config:
event_types = config['event_type']
if isinstance(event_types, six.string_types):
event_types = [event_types]
for t in event_types:
if t.startswith('!'):
self.excluded_types.append(t[1:])
else:
self.included_types.append(t)
else:
self.included_types.append('*')
if self.excluded_types and not self.included_types:
self.included_types.append('*')
if 'number' in config:
self.number = config['number']
else:
self.number = 1
if 'timestamp' in config:
self.timestamp = timex.parse(config['timestamp'])
else:
self.timestamp = None
self.map_distinguished_by = dict()
if 'map_distinguished_by' in config:
self.map_distinguished_by = config['map_distinguished_by']
self.traits = dict()
if 'traits' in config:
for trait, criterion in config['traits'].items():
self.traits[trait] = Criterion.get_from_expression(
criterion, trait)
def parse_email(email_body):
# the desired time format is: %Y-%m-%DT%H:%M:%S
#t is an array of relative time objects, time objects, day_objects, month objects, and year objects detected in text
t,email_text = timex.parse(email_body)
def findYear(t):
print "Year: " + str(t[0])
def findMonth(t):
print "Month: "+ str(t[1])
def findDay(t):
print "Day: "+ str(t[2])
def findHour(t):
print "Hour: " +str(t[3])
def findMinute(t):
print "Minute: "+ str(t[4])
def findSecond(t):
print "Second: " + str(t[5])
print "Parsed entities: Y:%s M:%s D:%s H:%s M:%s S:%s" % (t[0], t[1], t[2], t[3], t[4], t[5])
return t, email_text
def test_special_range_wrap_add(self):
result_begin = datetime.datetime(2014, 7, 31, 6, 0, 0, 0)
result_end = datetime.datetime(2014, 8, 1, 6, 0, 0, 0)
exp = timex.parse("day + 6h")
t = exp(timestamp=self.dt)
self.assertTrue(t.is_range)
self.assertEqual(t.begin, result_begin)
self.assertEqual(t.end, result_end)
def test_timerange_replace(self):
result_begin = datetime.datetime(2014, 8, 1, 6, 10, 23, 550)
result_end = datetime.datetime(2014, 8, 7, 6, 20, 0, 0)
exp = timex.parse("($test_thingy to $other) @ 6h")
t = exp(test_thingy=self.dt, other=self.other_dt)
self.assertTrue(t.is_range)
self.assertEqual(t.begin, result_begin)
self.assertEqual(t.end, result_end)
def print_parses(phrases):
print
for p in phrases:
print p
result = timex.parse(p.split())
for t in result:
display(t, ' ')
print
def test_special_range_wrap_sub(self):
result_begin = datetime.datetime(2014, 8, 1, 18, 0, 0, 0)
result_end = datetime.datetime(2014, 8, 2, 18, 0, 0, 0)
exp = timex.parse("day - 6h")
t = exp(timestamp=datetime.datetime(2014, 8, 1, 19, 45, 30, 225))
self.assertTrue(t.is_range)
self.assertEqual(t.begin, result_begin)
self.assertEqual(t.end, result_end)
def get_possible_days(email_body):
possible_days = []
parsed_results = parse(email_body)
assert (len(parsed_results[0]) == len(parsed_results[1]))
for i in range(len(parsed_results[0])):
possible_days.append(
('2014', parsed_results[1][i], parsed_results[0][i]))
return possible_days
def test_timerange_ambig_duration(self):
# Ambiguous durations are a bit of a hack to make timex syntax
# compatable with the (much less flexible) syntax for timeranges
# used for some OpenStack projects. (mdragon)
result_begin = datetime.datetime(2014, 8, 1, 2, 0, 0, 0)
result_end = datetime.datetime(2014, 8, 2, 2, 0, 0, 0)
exp = timex.parse("day @ 2")
t = exp(timestamp=self.dt)
self.assertTrue(t.is_range)
self.assertEqual(t.begin, result_begin)
self.assertEqual(t.end, result_end)
def get_possible_days(email_body):
possible_days = []
parsed_results = parse(email_body)
assert(len(parsed_results[0]) == len(parsed_results[1]))
for i in range(len(parsed_results[0])):
possible_days.append( ('2014',
parsed_results[1][i],
parsed_results[0][i]) )
return possible_days
def get_distinguishing_traits(self, event, matching_criteria):
dist_traits = dict()
for dt in self.distinguished_by:
d_expr = None
if isinstance(dt, collections.Mapping):
trait_name = dt.keys()[0]
d_expr = timex.parse(dt[trait_name])
else:
trait_name = dt
event_trait_name = matching_criteria.map_distinguished_by.get(trait_name, trait_name)
if d_expr is not None:
dist_traits[trait_name] = d_expr(timestamp=event[event_trait_name])
else:
dist_traits[trait_name] = event[event_trait_name]
return dist_traits
def get_distinguishing_traits(self, event, matching_criteria):
dist_traits = dict()
for dt in self.distinguished_by:
d_expr = None
if isinstance(dt, collections.Mapping):
trait_name = dt.keys()[0]
d_expr = timex.parse(dt[trait_name])
else:
trait_name = dt
event_trait_name = matching_criteria.map_distinguished_by.get(
trait_name, trait_name)
if d_expr is not None:
dist_traits[trait_name] = d_expr(
timestamp=event[event_trait_name])
else:
dist_traits[trait_name] = event[event_trait_name]
return dist_traits
def __init__(self, config, debug_manager):
if 'name' not in config:
raise DefinitionError("Required field in trigger definition not "
"specified 'name'")
self.name = config['name']
self.debug_level = int(config.get('debug_level', 0))
self.distinguished_by = config.get('distinguished_by', [])
for dt in self.distinguished_by:
if isinstance(dt, collections.Mapping):
if len(dt) > 1:
raise DefinitionError(
"Invalid distinguising expression "
"%s. Only one trait allowed in an expression" %
str(dt))
self.fire_delay = config.get('fire_delay', 0)
if 'expiration' not in config:
raise DefinitionError("Required field in trigger definition not "
"specified 'expiration'")
self.expiration = timex.parse(config['expiration'])
self.fire_pipeline = config.get('fire_pipeline')
self.expire_pipeline = config.get('expire_pipeline')
if not self.fire_pipeline and not self.expire_pipeline:
raise DefinitionError("At least one of: 'fire_pipeline' or "
"'expire_pipeline' must be specified in a "
"trigger definition.")
if 'fire_criteria' not in config:
raise DefinitionError(
"Required criteria in trigger definition not "
"specified 'fire_criteria'")
self.fire_criteria = [Criteria(c) for c in config['fire_criteria']]
if 'match_criteria' not in config:
raise DefinitionError(
"Required criteria in trigger definition not "
"specified 'match_criteria'")
self.match_criteria = [Criteria(c) for c in config['match_criteria']]
self.load_criteria = []
if 'load_criteria' in config:
self.load_criteria = [Criteria(c) for c in config['load_criteria']]
if debug_manager:
self.set_debugger(debug_manager)
def __init__(self, config, debug_manager):
if 'name' not in config:
raise DefinitionError("Required field in trigger definition not "
"specified 'name'")
self.name = config['name']
self.debug_level = int(config.get('debug_level', 0))
self.distinguished_by = config.get('distinguished_by', [])
for dt in self.distinguished_by:
if isinstance(dt, collections.Mapping):
if len(dt) > 1:
raise DefinitionError(
"Invalid distinguising expression "
"%s. Only one trait allowed in an expression"
% str(dt))
self.fire_delay = config.get('fire_delay', 0)
if 'expiration' not in config:
raise DefinitionError("Required field in trigger definition not "
"specified 'expiration'")
self.expiration = timex.parse(config['expiration'])
self.fire_pipeline = config.get('fire_pipeline')
self.expire_pipeline = config.get('expire_pipeline')
if not self.fire_pipeline and not self.expire_pipeline:
raise DefinitionError("At least one of: 'fire_pipeline' or "
"'expire_pipeline' must be specified in a "
"trigger definition.")
if 'fire_criteria' not in config:
raise DefinitionError("Required criteria in trigger definition "
"not specified 'fire_criteria'")
self.fire_criteria = [Criteria(c) for c in config['fire_criteria']]
if 'match_criteria' not in config:
raise DefinitionError("Required criteria in trigger definition "
"not specified 'match_criteria'")
self.match_criteria = [Criteria(c) for c in config['match_criteria']]
self.load_criteria = []
if 'load_criteria' in config:
self.load_criteria = [Criteria(c) for c in config['load_criteria']]
if debug_manager:
self.set_debugger(debug_manager)
def __init__(self, expression, trait_name):
self.trait_name = trait_name
self.time_expr = timex.parse(expression)
def test_var(self):
exp = timex.parse("$test_thingy")
t = exp(test_thingy=self.dt)
self.assertFalse(t.is_range)
self.assertEqual(t.timestamp, self.dt)
def __init__(self, expression, trait_name):
self.trait_name = trait_name
self.time_expr = timex.parse(expression)
def test_timestamp_sub(self):
result = datetime.datetime(2014, 7, 31, 0, 10, 23, 550)
exp = timex.parse("$test_thingy - 1d 2h")
t = exp(test_thingy=self.dt)
self.assertFalse(t.is_range)
self.assertEqual(t.timestamp, result)
def test_timestamp_replace(self):
result = datetime.datetime(2014, 8, 7, 6, 10, 23, 550)
exp = timex.parse("$test_thingy @ 7d 6h")
t = exp(test_thingy=self.dt)
self.assertFalse(t.is_range)
self.assertEqual(t.timestamp, result)
def __init__(self, config, db=None, pipeline_handlers=None,
pipeline_config=None, trigger_defs=None, time_sync=None,
proc_name='pipeline_worker'):
# name used to distinguish worker processes in logs
self.proc_name = proc_name
logger.debug("PipelineManager(%s): Using config: %s"
% (self.proc_name, str(config)))
config = ConfigManager.wrap(config, self.config_description())
self.config = config
self.trigger_definitions = []
config.check_config()
config.add_config_path(*config['config_path'])
if time_sync is None:
time_sync = ts.TimeSync()
self.time_sync = time_sync
if db is not None:
self.db = db
else:
self.db = DBInterface(config['database'])
if pipeline_handlers is not None:
self.pipeline_handlers = pipeline_handlers
else:
self.pipeline_handlers = self._load_plugins(
config['pipeline_handlers'])
logger.debug("Pipeline handlers: %s" % str(self.pipeline_handlers))
if pipeline_config is not None:
self.pipeline_config = pipeline_config
else:
self.pipeline_config = config.load_file(config['pipeline_config'])
logger.debug("Pipeline config: %s" % str(self.pipeline_config))
for pipeline, handler_configs in self.pipeline_config.items():
self.pipeline_config[pipeline] = [
Pipeline.check_handler_config(conf,
self.pipeline_handlers)
for conf in handler_configs]
if trigger_defs is not None:
self.trigger_definitions = trigger_defs
else:
# trigger_definition config file is optional
if config.contains('trigger_definitions'):
defs = config.load_file(config['trigger_definitions'])
logger.debug("Loaded trigger definitions %s" % str(defs))
self.trigger_definitions = [
TriggerDefinition(conf, None) for conf in defs]
self.trigger_manager = TriggerManager(
self.config, db=self.db,
trigger_defs=self.trigger_definitions,
time_sync=time_sync)
self.pipeline_worker_batch_size = config['pipeline_worker_batch_size']
self.pipeline_worker_delay = config['pipeline_worker_delay']
self.statistics_period = config['statistics_period']
self.purge_completed_streams = config['purge_completed_streams']
self.trim_events = config['trim_events']
self.trim_events_batch_size = config['trim_events_batch_size']
try:
self.trim_events_age = timex.parse(str(config['trim_events_age']))
except timex.TimexError:
logger.error("Invalid trim event expression: %s Event trimming "
"disabled." % config['trim_events_age'])
self.trim_events_age = None
self.trim_events = False
self.streams_fired = 0
self.streams_expired = 0
self.streams_loaded = 0
self.last_status = self.current_time()
def __init__(self,
config,
db=None,
pipeline_handlers=None,
pipeline_config=None,
trigger_defs=None,
time_sync=None,
proc_name='pipeline_worker'):
# name used to distinguish worker processes in logs
self.proc_name = proc_name
logger.debug("PipelineManager(%s): Using config: %s" %
(self.proc_name, str(config)))
config = ConfigManager.wrap(config, self.config_description())
self.config = config
self.trigger_definitions = []
config.check_config()
config.add_config_path(*config['config_path'])
if time_sync is None:
time_sync = ts.TimeSync()
self.time_sync = time_sync
if db is not None:
self.db = db
else:
self.db = DBInterface(config['database'])
if pipeline_handlers is not None:
self.pipeline_handlers = pipeline_handlers
else:
self.pipeline_handlers = self._load_plugins(
config['pipeline_handlers'])
logger.debug("Pipeline handlers: %s" % str(self.pipeline_handlers))
if pipeline_config is not None:
self.pipeline_config = pipeline_config
else:
self.pipeline_config = config.load_file(config['pipeline_config'])
logger.debug("Pipeline config: %s" % str(self.pipeline_config))
for pipeline, handler_configs in self.pipeline_config.items():
self.pipeline_config[pipeline] = [
Pipeline.check_handler_config(conf, self.pipeline_handlers)
for conf in handler_configs
]
if trigger_defs is not None:
self.trigger_definitions = trigger_defs
else:
# trigger_definition config file is optional
if config.contains('trigger_definitions'):
defs = config.load_file(config['trigger_definitions'])
logger.debug("Loaded trigger definitions %s" % str(defs))
self.trigger_definitions = [
TriggerDefinition(conf, None) for conf in defs
]
self.trigger_manager = TriggerManager(
self.config,
db=self.db,
trigger_defs=self.trigger_definitions,
time_sync=time_sync)
self.pipeline_worker_batch_size = config['pipeline_worker_batch_size']
self.pipeline_worker_delay = config['pipeline_worker_delay']
self.statistics_period = config['statistics_period']
self.purge_completed_streams = config['purge_completed_streams']
self.trim_events = config['trim_events']
self.trim_events_batch_size = config['trim_events_batch_size']
try:
self.trim_events_age = timex.parse(str(config['trim_events_age']))
except timex.TimexError:
logger.error("Invalid trim event expression: %s Event trimming "
"disabled." % config['trim_events_age'])
self.trim_events_age = None
self.trim_events = False
self.streams_fired = 0
self.streams_expired = 0
self.streams_loaded = 0
self.last_status = self.current_time()
def test_timerange(self):
exp = timex.parse("$test_thingy to $other")
t = exp(test_thingy=self.dt, other=self.other_dt)
self.assertTrue(t.is_range)
self.assertEqual(t.begin, self.dt)
self.assertEqual(t.end, self.other_dt)