def run(self):
if 'logging_config' in self.config_mgr:
fileConfig(self.config_mgr['logging_config'])
else:
logging.basicConfig()
if 'log_level' in self.config_mgr:
level = self.config_mgr['log_level']
level = getattr(logging, level.upper())
logging.getLogger('winchester').setLevel(level)
self.pm_lock = threading.Lock()
self.pipe = PipelineManager(self.config_mgr)
# TODO add trigger defs from the DB at startup
# start threads
self.stream_def_thread = threading.Thread(name='stream_defs_pipe',
target=pipe_stream_definition_consumer,
args=(self.kafka_config, self.pm_lock, self.pipe,))
self.pipeline_ready_thread = threading.Thread(name='pipeline',
target=self.pipeline_ready_processor,
args=(self.pm_lock, self.pipe,))
self.stream_def_thread.start()
self.pipeline_ready_thread.start()
self.stream_def_thread.join()
self.pipeline_ready_thread.join()
log.debug('Exiting')
def __init__(self, conf):
super(PipelineProcessor, self).__init__(conf)
self._winchester_config = conf.winchester.winchester_config
self._config_mgr = ConfigManager.load_config_file(
self._winchester_config)
self._group = conf.kafka.stream_def_pipe_group
self._pm_lock = threading.Lock()
self._pipe = PipelineManager(self._config_mgr)
def main():
parser = argparse.ArgumentParser(description="Winchester pipeline worker")
parser.add_argument('--config', '-c', default='winchester.yaml',
help='The name of the winchester config file')
parser.add_argument('--name', '-n', default='pipeline_worker',
help='The name of this process for logging purposes')
parser.add_argument('--daemon', '-d', help='Run in daemon mode.')
args = parser.parse_args()
conf = ConfigManager.load_config_file(args.config)
proc_name = args.name
if 'log_level' in conf:
level = conf['log_level']
level = getattr(logging, level.upper())
else:
level = logging.INFO
if 'log_file' in conf:
log_file = conf['log_file'] % dict(proc_name=proc_name)
else:
log_file = '%(proc_name)s.log' % dict(proc_name=proc_name)
# This is a hack, but it's needed to pass the logfile name & default
# loglevel into log handlers configured with a config file. (mdragon)
logging.LOCAL_LOG_FILE = log_file
logging.LOCAL_DEFAULT_LEVEL = level
if 'logging_config' in conf:
fileConfig(conf['logging_config'])
else:
logging.basicConfig()
logging.getLogger('winchester').setLevel(level)
timesync = time_sync.TimeSync(conf)
pipe = PipelineManager(conf, time_sync=timesync, proc_name=proc_name)
if args.daemon:
print("Backgrounding for daemon mode.")
with daemon.DaemonContext():
pipe.run()
else:
pipe.run()
def main():
parser = argparse.ArgumentParser(description="Winchester pipeline worker")
parser.add_argument('--config',
'-c',
default='winchester.yaml',
help='The name of the winchester config file')
parser.add_argument('--daemon', '-d', help='Run in daemon mode.')
args = parser.parse_args()
conf = ConfigManager.load_config_file(args.config)
if 'logging_config' in conf:
fileConfig(conf['logging_config'])
else:
logging.basicConfig()
if 'log_level' in conf:
level = conf['log_level']
level = getattr(logging, level.upper())
logging.getLogger('winchester').setLevel(level)
pipe = PipelineManager(conf)
if args.daemon:
print "Backgrounding for daemon mode."
with daemon.DaemonContext():
pipe.run()
else:
pipe.run()
def main():
parser = argparse.ArgumentParser(description="Winchester pipeline worker")
parser.add_argument('--config', '-c', default='winchester.yaml',
help='The name of the winchester config file')
parser.add_argument('--daemon', '-d', help='Run in daemon mode.')
args = parser.parse_args()
conf = ConfigManager.load_config_file(args.config)
if 'logging_config' in conf:
fileConfig(conf['logging_config'])
else:
logging.basicConfig()
if 'log_level' in conf:
level = conf['log_level']
level = getattr(logging, level.upper())
logging.getLogger('winchester').setLevel(level)
pipe = PipelineManager(conf)
if args.daemon:
print "Backgrounding for daemon mode."
with daemon.DaemonContext():
pipe.run()
else:
pipe.run()
def main():
parser = argparse.ArgumentParser(description="Winchester pipeline worker")
parser.add_argument('--config',
'-c',
default='winchester.yaml',
help='The name of the winchester config file')
parser.add_argument('--name',
'-n',
default='pipeline_worker',
help='The name of this process for logging purposes')
parser.add_argument('--daemon', '-d', help='Run in daemon mode.')
args = parser.parse_args()
conf = ConfigManager.load_config_file(args.config)
proc_name = args.name
if 'log_level' in conf:
level = conf['log_level']
level = getattr(logging, level.upper())
else:
level = logging.INFO
if 'log_file' in conf:
log_file = conf['log_file'] % dict(proc_name=proc_name)
else:
log_file = '%(proc_name)s.log' % dict(proc_name=proc_name)
# This is a hack, but it's needed to pass the logfile name & default
# loglevel into log handlers configured with a config file. (mdragon)
logging.LOCAL_LOG_FILE = log_file
logging.LOCAL_DEFAULT_LEVEL = level
if 'logging_config' in conf:
fileConfig(conf['logging_config'])
else:
logging.basicConfig()
logging.getLogger('winchester').setLevel(level)
timesync = time_sync.TimeSync(conf)
pipe = PipelineManager(conf, time_sync=timesync, proc_name=proc_name)
if args.daemon:
print("Backgrounding for daemon mode.")
with daemon.DaemonContext():
pipe.run()
else:
pipe.run()
class TriggerTest():
""" Trigger Test
Adds Stream Definitions to the TriggerManager and PipelineManager classes. Adds Fake
distilled events to the TriggerManager and ensures the Fire and expire handlers will get called.
This test uses the winchester mysql DB.
"""
""" test data """
trig_def_fc1 = [{'distinguished_by': ['instance_id'],
'fire_criteria': [{'event_type': 'compute.instance.create.start'},
{'event_type': 'compute.instance.create.end'}],
'match_criteria': [{'event_type': 'compute.instance.create.*'}],
'name': 'fc1_trigger',
'debug_level': 2,
'expiration': '$last + 1h',
'fire_pipeline': 'test_pipeline',
'expire_pipeline': 'test_expire_pipeline'}]
trig_def_fc1_tenant406904_filter = [{'distinguished_by': ['instance_id'],
'fire_criteria': [{'event_type': 'compute.instance.create.start'}, {'event_type': 'compute.instance.create.end'}],
'match_criteria': [{'traits': {'tenant_id': '406904'}, 'event_type': 'compute.instance.create.*'}],
'name': 'trig_def_fc1_406904',
'debug_level': 2,
'expiration': '$first + 10s',
'fire_pipeline': 'test_pipeline',
'expire_pipeline': 'test_expire_pipeline'}]
trig_def_fc1_tenant123456_filter = [{'distinguished_by': ['instance_id'],
'fire_criteria': [{'event_type': 'compute.instance.create.start'}, {'event_type': 'compute.instance.create.end'}],
'match_criteria': [{'traits': {'tenant_id': '123456'},
'event_type': 'compute.instance.create.*'}],
'name': 'fc1_trigger_123456',
'debug_level': 2,
'expiration': '$last + 24h',
'fire_pipeline': 'test_pipeline',
'expire_pipeline': 'test_expire_pipeline'}]
""" test adding events to cause fire criteria """
distilled_events_fc1_tenant_406904 = [{'os_distro': 'com.ubuntu',
'event_type': 'compute.instance.create.start',
'service': 'publisher-302689',
'instance_type': '512MB Standard Instance',
'tenant_id': '406904',
'instance_flavor_id': '2',
'hostname': 'server-462185',
'host': 'publisher-302689',
'instance_flavor': '512MB Standard Instance',
'instance_id': '111-3b0f-4057-b377-b65131e8532e',
'os_version': '12.04',
'state': 'building',
'os_architecture': 'x64',
'timestamp': datetime.utcnow(),
'request_id': 'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'message_id': '19701f6c-f51f-4ecb-85fb-7db40277627d'},
{'os_distro': 'com.ubuntu',
'message_id': '2ae21707-70ae-48a2-89c0-b08b11dc0b1a',
'service': 'publisher-302689',
'instance_type': '512MB Standard Instance',
'tenant_id': '406904',
'instance_flavor_id': '2',
'hostname': 'server-462185',
'host': 'publisher-302689',
'instance_flavor': '512MB Standard Instance',
'instance_id': '111-3b0f-4057-b377-b65131e8532e',
'os_version': '12.04',
'state': 'active',
'os_architecture': 'x64',
'timestamp': datetime.utcnow(),
'request_id': 'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'launched_at': datetime.utcnow(),
'event_type': 'compute.instance.create.end'}]
distilled_events_fc1_tenant_406904_missing_end = [{'os_distro': 'com.ubuntu',
'event_type': 'compute.instance.create.start',
'service': 'publisher-302689',
'instance_type': '512MB Standard Instance',
'tenant_id': '406904',
'instance_flavor_id': '2',
'hostname': 'server-462185',
'host': 'publisher-302689',
'instance_flavor': '512MB Standard Instance',
'instance_id': '333-3b0f-4057-b377-b65131e8532e',
'os_version': '12.04',
'state': 'building',
'os_architecture': 'x64',
'timestamp': datetime.utcnow(),
'request_id': 'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'message_id': '19701f6c-f51f-4ecb-85fb-7db40277627d'}]
distilled_events_fc1_tenant_123456 = [{'os_distro': 'com.ubuntu',
'event_type': 'compute.instance.create.start',
'service': 'publisher-302689',
'instance_type': '512MB Standard Instance',
'tenant_id': '123456',
'instance_flavor_id': '2',
'hostname': 'server-462185',
'host': 'publisher-302689',
'instance_flavor': '512MB Standard Instance',
'instance_id': '456-3b0f-4057-b377-b65131e8532e',
'os_version': '12.04',
'state': 'building',
'os_architecture': 'x64',
'timestamp': datetime.utcnow(),
'request_id': 'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'message_id': '19701f6c-f51f-4ecb-85fb-7db40277627d'},
{'os_distro': 'com.ubuntu',
'message_id': '2ae21707-70ae-48a2-89c0-b08b11dc0b1a',
'service': 'publisher-302689',
'instance_type': '512MB Standard Instance',
'tenant_id': '123456',
'instance_flavor_id': '2',
'hostname': 'server-462185',
'host': 'publisher-302689',
'instance_flavor': '512MB Standard Instance',
'instance_id': '456-3b0f-4057-b377-b65131e8532e',
'os_version': '12.04',
'state': 'active',
'os_architecture': 'x64',
'timestamp': datetime.utcnow(),
'request_id': 'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'launched_at': datetime.utcnow(),
'event_type': 'compute.instance.create.end'}]
""" trigger defs for fire criteria 2 - looking for exists"""
trig_def_fc2_rackspace_billing = [{'distinguished_by': ['instance_id',
{'timestamp': 'day'}],
'fire_criteria': [{'event_type': 'compute.instance.exists'}],
'match_criteria': [{'event_type': ['compute.instance.*',
'!compute.instance.exists']},
{'event_type': 'compute.instance.exists',
'map_distingushed_by': {'timestamp': 'audit_period_beginning'}}],
'name': 'rackspace_billing',
'debug_level': 2,
'expiration': '$last + 1h',
'fire_pipeline': 'test_pipeline',
'expire_pipeline': 'test_expire_pipeline'}]
trig_def_fc3_rackspace = [{'distinguished_by': ['instance_id', {'timestamp': 'day'}],
'fire_criteria': [{'traits': {'audit_period_ending': {'datetime': '$audit_period_beginning + 1d'}},
'event_type': 'compute.instance.exists'}],
'match_criteria': [{'event_type': ['compute.instance.*',
'snapshot_instance',
'keypair.import.*',
'rebuild_instance',
'compute.instance.*',
'!compute.instance.exists',
'!compute.instance.exists.failed',
'!compute.instance.exists.warnings',
'!compute.instance.exists.verified']},
{'event_type': 'compute.instance.exists',
'map_distinguished_by': {'timestamp': 'audit_period_beginning'}}],
'name': 'rackspace_test_trigger',
'debug_level': 2,
'expiration': '$last + 2d',
'fire_pipeline': 'test_fire_pipeline',
'expire_pipeline': 'test_expire_pipeline'}]
distilled_events_fc2_tenant_222333 = [{'os_distro': 'com.ubuntu',
'event_type': 'compute.instance.create.start',
'service': 'publisher-302689',
'instance_type': '512MB Standard Instance',
'tenant_id': '222333',
'instance_flavor_id': '2',
'hostname': 'server-462185',
'host': 'publisher-302689',
'instance_flavor': '512MB Standard Instance',
'instance_id': '772b2f73-3b0f-4057-b377-b65131e8532e',
'os_version': '12.04',
'state': 'building',
'os_architecture': 'x64',
'timestamp': datetime.utcnow(),
'request_id': 'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'message_id': '19701f6c-f51f-4ecb-85fb-7db40277627d'},
{'os_distro': 'com.ubuntu',
'message_id': '2ae21707-70ae-48a2-89c0-b08b11dc0b1a',
'service': 'publisher-302689',
'instance_type': '512MB Standard Instance',
'tenant_id': '222333',
'instance_flavor_id': '2',
'hostname': 'server-462185',
'host': 'publisher-302689',
'instance_flavor': '512MB Standard Instance',
'instance_id': '772b2f73-3b0f-4057-b377-b65131e8532e',
'os_version': '12.04',
'state': 'active',
'os_architecture': 'x64',
'timestamp': datetime.utcnow(),
'request_id': 'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'launched_at': datetime.utcnow(),
'event_type': 'compute.instance.create.end'},
{'os_distro': 'com.ubuntu',
'message_id': '2ae21707-70ae-48a2-89c0-b08b11dc0b1a',
'service': 'publisher-302689',
'instance_type': '512MB Standard Instance',
'tenant_id': '222333',
'instance_flavor_id': '2',
'hostname': 'server-462185',
'host': 'publisher-302689',
'instance_flavor': '512MB Standard Instance',
'instance_id': '772b2f73-3b0f-4057-b377-b65131e8532e',
'os_version': '12.04',
'state': 'active',
'os_architecture': 'x64',
'timestamp': datetime.utcnow(),
'request_id': 'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'launched_at': datetime.utcnow(),
'event_type': 'compute.instance.exists'}]
def __init__(self, winchester_config):
self.winchester_config = winchester_config
self.config = ConfigManager.load_config_file(winchester_config)
self.trigger_manager = TriggerManager(self.config)
self.pipe = PipelineManager(self.config)
def _add_unique_event(self, e):
''' make the static test data contain unique message id's '''
e['message_id'] = uuid.uuid4()
self.trigger_manager.add_event(e)
def add_test_stream_definitions(self):
self.trigger_manager.add_trigger_definition(
TriggerTest.trig_def_fc1_tenant406904_filter)
self.trigger_manager.add_trigger_definition(
TriggerTest.trig_def_fc1_tenant123456_filter)
self.pipe.add_trigger_definition(
TriggerTest.trig_def_fc1_tenant406904_filter)
self.pipe.add_trigger_definition(
TriggerTest.trig_def_fc1_tenant123456_filter)
def add_distilled_events_to_fire(self):
for e in TriggerTest.distilled_events_fc1_tenant_406904:
self._add_unique_event(e)
for e in TriggerTest.distilled_events_fc1_tenant_123456:
self._add_unique_event(e)
def add_distilled_events_to_expire(self):
for e in TriggerTest.distilled_events_fc1_tenant_406904_missing_end:
self._add_unique_event(e)
def add_distilled_events_with_no_match(self):
for e in TriggerTest.distilled_events_fc2_tenant_222333:
self._add_unique_event(e)
def check_for_expired_streams(self):
stream_count = self.pipe.process_ready_streams(self.pipe.pipeline_worker_batch_size,
expire=True)
return stream_count
def check_for_fired_streams(self):
stream_count = self.pipe.process_ready_streams(
self.pipe.pipeline_worker_batch_size)
return stream_count
def test_no_match(self):
self.add_distilled_events_with_no_match()
time.sleep(2)
fired_count = self.check_for_fired_streams()
expired_count = self.check_for_expired_streams()
if (fired_count == 0 and expired_count == 0):
print ("test_no_match: Success")
else:
print ("test_no_match: Failed")
def test_fired(self):
self.add_distilled_events_to_fire()
time.sleep(3)
fired_count = self.check_for_fired_streams()
expired_count = self.check_for_expired_streams()
if (expired_count == 0 and fired_count == 2):
print ("test_fired: Success")
else:
print ("test_fired: Failed")
def test_expired(self):
self.add_distilled_events_to_expire()
time.sleep(11)
fired_count = self.check_for_fired_streams()
expired_count = self.check_for_expired_streams()
if (expired_count == 1 and fired_count == 0):
print ("test_expired: Success")
else:
print ("test_expired: Failed")
def __init__(self, winchester_config):
self.winchester_config = winchester_config
self.config = ConfigManager.load_config_file(winchester_config)
self.trigger_manager = TriggerManager(self.config)
self.pipe = PipelineManager(self.config)
class PipelineProcessor(EventProcessorBase):
"""Pipeline Processor
PipelineProcessor uses the stacktach PipelineManager to
load pipeline handlers, and process ready and expired streams.
The PipelineManager contains a TriggerManager so that
handlers can optionally add more events to the TriggerManager
filtered stream. The TriggerManager within the PipelineManager
is initialized with stream definitions dynamically.
"""
def __init__(self, conf):
super(PipelineProcessor, self).__init__(conf)
self._winchester_config = conf.winchester.winchester_config
self._config_mgr = ConfigManager.load_config_file(
self._winchester_config)
self._group = conf.kafka.stream_def_pipe_group
self._pm_lock = threading.Lock()
self._pipe = PipelineManager(self._config_mgr)
def run(self):
# read stream-definitions from DB at startup and add
stream_defs = self.stream_defs_from_database()
if len(stream_defs) > 0:
log.debug(
'Loading {} stream definitions from the DB at startup'.format(
len(stream_defs)))
self._pipe.add_trigger_definition(stream_defs)
# start threads
self.stream_def_thread = threading.Thread(
name='stream_defs_pipe',
target=self.stream_definition_consumer,
args=(self.conf, self._pm_lock, self._group, self._pipe,))
self.pipeline_ready_thread = threading.Thread(
name='pipeline',
target=self.pipeline_ready_processor,
args=(self._pm_lock, self._pipe,))
log.debug('Starting stream_defs_pipe and pipeline threads')
self.stream_def_thread.start()
self.pipeline_ready_thread.start()
self.stream_def_thread.join()
self.pipeline_ready_thread.join()
log.debug('Exiting')
def pipeline_ready_processor(self, lock, pipe):
statsd = monascastatsd.Client(name='monasca',
dimensions=self.dimensions)
fired_streams = statsd.get_counter('fired_streams')
expired_streams = statsd.get_counter('expired_streams')
while True:
lock.acquire()
try:
fire_ct = pipe.process_ready_streams(
pipe.pipeline_worker_batch_size)
expire_ct = pipe.process_ready_streams(
pipe.pipeline_worker_batch_size,
expire=True)
if fire_ct > 0:
fired_streams.increment()
if expire_ct > 0:
expired_streams.increment()
except Exception as e:
log.exception(e)
finally:
lock.release()
if (pipe.current_time() -
pipe.last_status).seconds > pipe.statistics_period:
pipe._log_statistics()
if not fire_ct and not expire_ct:
log.debug("No streams to fire or expire. Sleeping...")
time.sleep(pipe.pipeline_worker_delay)
else:
log.debug(
"Fired {} streams, Expired {} streams".format(
fire_ct,
expire_ct))
def __init__(self, winchester_config):
self.winchester_config = winchester_config
self.config = ConfigManager.load_config_file(winchester_config)
self.trigger_manager = TriggerManager(self.config)
self.pipe = PipelineManager(self.config)
class TriggerTest():
""" Trigger Test
Adds Stream Definitions to the TriggerManager and PipelineManager classes. Adds Fake
distilled events to the TriggerManager and ensures the Fire and expire handlers will get called.
This test uses the winchester mysql DB.
"""
""" test data """
trig_def_fc1 = [{
'distinguished_by': ['instance_id'],
'fire_criteria': [{
'event_type': 'compute.instance.create.start'
}, {
'event_type': 'compute.instance.create.end'
}],
'match_criteria': [{
'event_type': 'compute.instance.create.*'
}],
'name':
'fc1_trigger',
'debug_level':
2,
'expiration':
'$last + 1h',
'fire_pipeline':
'test_pipeline',
'expire_pipeline':
'test_expire_pipeline'
}]
trig_def_fc1_tenant406904_filter = [{
'distinguished_by': ['instance_id'],
'fire_criteria': [{
'event_type': 'compute.instance.create.start'
}, {
'event_type': 'compute.instance.create.end'
}],
'match_criteria': [{
'traits': {
'tenant_id': '406904'
},
'event_type': 'compute.instance.create.*'
}],
'name':
'trig_def_fc1_406904',
'debug_level':
2,
'expiration':
'$first + 10s',
'fire_pipeline':
'test_pipeline',
'expire_pipeline':
'test_expire_pipeline'
}]
trig_def_fc1_tenant123456_filter = [{
'distinguished_by': ['instance_id'],
'fire_criteria': [{
'event_type': 'compute.instance.create.start'
}, {
'event_type': 'compute.instance.create.end'
}],
'match_criteria': [{
'traits': {
'tenant_id': '123456'
},
'event_type': 'compute.instance.create.*'
}],
'name':
'fc1_trigger_123456',
'debug_level':
2,
'expiration':
'$last + 24h',
'fire_pipeline':
'test_pipeline',
'expire_pipeline':
'test_expire_pipeline'
}]
""" test adding events to cause fire criteria """
distilled_events_fc1_tenant_406904 = [{
'os_distro':
'com.ubuntu',
'event_type':
'compute.instance.create.start',
'service':
'publisher-302689',
'instance_type':
'512MB Standard Instance',
'tenant_id':
'406904',
'instance_flavor_id':
'2',
'hostname':
'server-462185',
'host':
'publisher-302689',
'instance_flavor':
'512MB Standard Instance',
'instance_id':
'111-3b0f-4057-b377-b65131e8532e',
'os_version':
'12.04',
'state':
'building',
'os_architecture':
'x64',
'timestamp':
datetime.utcnow(),
'request_id':
'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'message_id':
'19701f6c-f51f-4ecb-85fb-7db40277627d'
}, {
'os_distro':
'com.ubuntu',
'message_id':
'2ae21707-70ae-48a2-89c0-b08b11dc0b1a',
'service':
'publisher-302689',
'instance_type':
'512MB Standard Instance',
'tenant_id':
'406904',
'instance_flavor_id':
'2',
'hostname':
'server-462185',
'host':
'publisher-302689',
'instance_flavor':
'512MB Standard Instance',
'instance_id':
'111-3b0f-4057-b377-b65131e8532e',
'os_version':
'12.04',
'state':
'active',
'os_architecture':
'x64',
'timestamp':
datetime.utcnow(),
'request_id':
'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'launched_at':
datetime.utcnow(),
'event_type':
'compute.instance.create.end'
}]
distilled_events_fc1_tenant_406904_missing_end = [{
'os_distro':
'com.ubuntu',
'event_type':
'compute.instance.create.start',
'service':
'publisher-302689',
'instance_type':
'512MB Standard Instance',
'tenant_id':
'406904',
'instance_flavor_id':
'2',
'hostname':
'server-462185',
'host':
'publisher-302689',
'instance_flavor':
'512MB Standard Instance',
'instance_id':
'333-3b0f-4057-b377-b65131e8532e',
'os_version':
'12.04',
'state':
'building',
'os_architecture':
'x64',
'timestamp':
datetime.utcnow(),
'request_id':
'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'message_id':
'19701f6c-f51f-4ecb-85fb-7db40277627d'
}]
distilled_events_fc1_tenant_123456 = [{
'os_distro':
'com.ubuntu',
'event_type':
'compute.instance.create.start',
'service':
'publisher-302689',
'instance_type':
'512MB Standard Instance',
'tenant_id':
'123456',
'instance_flavor_id':
'2',
'hostname':
'server-462185',
'host':
'publisher-302689',
'instance_flavor':
'512MB Standard Instance',
'instance_id':
'456-3b0f-4057-b377-b65131e8532e',
'os_version':
'12.04',
'state':
'building',
'os_architecture':
'x64',
'timestamp':
datetime.utcnow(),
'request_id':
'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'message_id':
'19701f6c-f51f-4ecb-85fb-7db40277627d'
}, {
'os_distro':
'com.ubuntu',
'message_id':
'2ae21707-70ae-48a2-89c0-b08b11dc0b1a',
'service':
'publisher-302689',
'instance_type':
'512MB Standard Instance',
'tenant_id':
'123456',
'instance_flavor_id':
'2',
'hostname':
'server-462185',
'host':
'publisher-302689',
'instance_flavor':
'512MB Standard Instance',
'instance_id':
'456-3b0f-4057-b377-b65131e8532e',
'os_version':
'12.04',
'state':
'active',
'os_architecture':
'x64',
'timestamp':
datetime.utcnow(),
'request_id':
'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'launched_at':
datetime.utcnow(),
'event_type':
'compute.instance.create.end'
}]
""" trigger defs for fire criteria 2 - looking for exists"""
trig_def_fc2_rackspace_billing = [{
'distinguished_by': ['instance_id', {
'timestamp': 'day'
}],
'fire_criteria': [{
'event_type': 'compute.instance.exists'
}],
'match_criteria': [{
'event_type': ['compute.instance.*', '!compute.instance.exists']
}, {
'event_type': 'compute.instance.exists',
'map_distingushed_by': {
'timestamp': 'audit_period_beginning'
}
}],
'name':
'rackspace_billing',
'debug_level':
2,
'expiration':
'$last + 1h',
'fire_pipeline':
'test_pipeline',
'expire_pipeline':
'test_expire_pipeline'
}]
trig_def_fc3_rackspace = [{
'distinguished_by': ['instance_id', {
'timestamp': 'day'
}],
'fire_criteria': [{
'traits': {
'audit_period_ending': {
'datetime': '$audit_period_beginning + 1d'
}
},
'event_type': 'compute.instance.exists'
}],
'match_criteria': [{
'event_type': [
'compute.instance.*', 'snapshot_instance', 'keypair.import.*',
'rebuild_instance', 'compute.instance.*',
'!compute.instance.exists', '!compute.instance.exists.failed',
'!compute.instance.exists.warnings',
'!compute.instance.exists.verified'
]
}, {
'event_type': 'compute.instance.exists',
'map_distinguished_by': {
'timestamp': 'audit_period_beginning'
}
}],
'name':
'rackspace_test_trigger',
'debug_level':
2,
'expiration':
'$last + 2d',
'fire_pipeline':
'test_fire_pipeline',
'expire_pipeline':
'test_expire_pipeline'
}]
distilled_events_fc2_tenant_222333 = [{
'os_distro':
'com.ubuntu',
'event_type':
'compute.instance.create.start',
'service':
'publisher-302689',
'instance_type':
'512MB Standard Instance',
'tenant_id':
'222333',
'instance_flavor_id':
'2',
'hostname':
'server-462185',
'host':
'publisher-302689',
'instance_flavor':
'512MB Standard Instance',
'instance_id':
'772b2f73-3b0f-4057-b377-b65131e8532e',
'os_version':
'12.04',
'state':
'building',
'os_architecture':
'x64',
'timestamp':
datetime.utcnow(),
'request_id':
'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'message_id':
'19701f6c-f51f-4ecb-85fb-7db40277627d'
}, {
'os_distro':
'com.ubuntu',
'message_id':
'2ae21707-70ae-48a2-89c0-b08b11dc0b1a',
'service':
'publisher-302689',
'instance_type':
'512MB Standard Instance',
'tenant_id':
'222333',
'instance_flavor_id':
'2',
'hostname':
'server-462185',
'host':
'publisher-302689',
'instance_flavor':
'512MB Standard Instance',
'instance_id':
'772b2f73-3b0f-4057-b377-b65131e8532e',
'os_version':
'12.04',
'state':
'active',
'os_architecture':
'x64',
'timestamp':
datetime.utcnow(),
'request_id':
'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'launched_at':
datetime.utcnow(),
'event_type':
'compute.instance.create.end'
}, {
'os_distro':
'com.ubuntu',
'message_id':
'2ae21707-70ae-48a2-89c0-b08b11dc0b1a',
'service':
'publisher-302689',
'instance_type':
'512MB Standard Instance',
'tenant_id':
'222333',
'instance_flavor_id':
'2',
'hostname':
'server-462185',
'host':
'publisher-302689',
'instance_flavor':
'512MB Standard Instance',
'instance_id':
'772b2f73-3b0f-4057-b377-b65131e8532e',
'os_version':
'12.04',
'state':
'active',
'os_architecture':
'x64',
'timestamp':
datetime.utcnow(),
'request_id':
'req-d096b6de-f451-4d00-bff0-646a8c8a23c3',
'launched_at':
datetime.utcnow(),
'event_type':
'compute.instance.exists'
}]
def __init__(self, winchester_config):
self.winchester_config = winchester_config
self.config = ConfigManager.load_config_file(winchester_config)
self.trigger_manager = TriggerManager(self.config)
self.pipe = PipelineManager(self.config)
def _add_unique_event(self, e):
''' make the static test data contain unique message id's '''
e['message_id'] = uuid.uuid4()
self.trigger_manager.add_event(e)
def add_test_stream_definitions(self):
self.trigger_manager.add_trigger_definition(
TriggerTest.trig_def_fc1_tenant406904_filter)
self.trigger_manager.add_trigger_definition(
TriggerTest.trig_def_fc1_tenant123456_filter)
self.pipe.add_trigger_definition(
TriggerTest.trig_def_fc1_tenant406904_filter)
self.pipe.add_trigger_definition(
TriggerTest.trig_def_fc1_tenant123456_filter)
def add_distilled_events_to_fire(self):
for e in TriggerTest.distilled_events_fc1_tenant_406904:
self._add_unique_event(e)
for e in TriggerTest.distilled_events_fc1_tenant_123456:
self._add_unique_event(e)
def add_distilled_events_to_expire(self):
for e in TriggerTest.distilled_events_fc1_tenant_406904_missing_end:
self._add_unique_event(e)
def add_distilled_events_with_no_match(self):
for e in TriggerTest.distilled_events_fc2_tenant_222333:
self._add_unique_event(e)
def check_for_expired_streams(self):
stream_count = self.pipe.process_ready_streams(
self.pipe.pipeline_worker_batch_size, expire=True)
return stream_count
def check_for_fired_streams(self):
stream_count = self.pipe.process_ready_streams(
self.pipe.pipeline_worker_batch_size)
return stream_count
def test_no_match(self):
self.add_distilled_events_with_no_match()
time.sleep(2)
fired_count = self.check_for_fired_streams()
expired_count = self.check_for_expired_streams()
if (fired_count == 0 and expired_count == 0):
print("test_no_match: Success")
else:
print("test_no_match: Failed")
def test_fired(self):
self.add_distilled_events_to_fire()
time.sleep(3)
fired_count = self.check_for_fired_streams()
expired_count = self.check_for_expired_streams()
if (expired_count == 0 and fired_count == 2):
print("test_fired: Success")
else:
print("test_fired: Failed")
def test_expired(self):
self.add_distilled_events_to_expire()
time.sleep(11)
fired_count = self.check_for_fired_streams()
expired_count = self.check_for_expired_streams()
if (expired_count == 1 and fired_count == 0):
print("test_expired: Success")
else:
print("test_expired: Failed")
