def schedule_add_activity(self, executionDate: str, username: str,
password: str, userId: str, courtId: str,
fromDate: str, toDate: str):
scheduler = BackgroundScheduler()
scheduler.add_executor('processpool')
date = dateutil.parser.isoparse(executionDate)
job = scheduler.add_job(
self.get_token_and_create_activity,
'date',
run_date=date,
args=[username, password, userId, courtId, fromDate, toDate])
scheduler.start()
class Filament(object):
"""Filament initialization and execution engine.
Filaments are lightweight Python modules which run
on top of Fibratus. They are often used to enrich/extend the
functionality of Fibratus by performing any type of logic
(aggregations, groupings, filters, counters, etc) on the
kernel event stream.
"""
def __init__(self):
"""Builds a new instance of the filament.
Attributes:
----------
filament_module: module
module which contains the filament logic
"""
self._filament_module = None
self._name = None
self._filters = []
self._cols = []
self._tabular = None
self._limit = 10
self._interval = 1
self._sort_by = None
self._sort_desc = True
self._logger = None
self._ansi_term = AnsiTerm()
self.scheduler = BackgroundScheduler()
def load_filament(self, name):
"""Loads the filament module.
Finds and loads the python module which
holds the filament logic. It also looks up for
some essential filament methods and raises an error
if they can't be found.
Parameters
----------
name: str
name of the filament to load
"""
self._name = name
Filament._assert_root_dir()
filament_path = self._find_filament_path(name)
if filament_path:
loader = SourceFileLoader(name, filament_path)
self._filament_module = loader.load_module()
sys.path.append(FILAMENTS_DIR)
doc = inspect.getdoc(self._filament_module)
if not doc:
raise FilamentError('Please provide a short '
'description for the filament')
on_next_kevent = self._find_filament_func('on_next_kevent')
if on_next_kevent:
if self._num_args(on_next_kevent) != 1:
raise FilamentError('Missing one argument on_next_kevent '
'method on filament')
self._initialize_funcs()
else:
raise FilamentError('Missing required on_next_kevent '
'method on filament')
else:
raise FilamentError('%s filament not found' % name)
def _initialize_funcs(self):
"""Setup the filament modules functions.
Functions
---------
set_filter: func
accepts the comma separated list of kernel events
for whose the filter should be applied
set_interval: func
establishes the fixed repeating interval in seconds
columns: func
configure the column set for the table
add_row: func
adds a new row to the table
sort_by: func
sorts the table by specific column
"""
def set_filter(*args):
self._filters = args
self._filament_module.set_filter = set_filter
def set_interval(interval):
if not type(interval) is int:
raise FilamentError('Interval must be an integer value')
self._interval = interval
self._filament_module.set_interval = set_interval
def columns(cols):
if not isinstance(cols, list):
raise FilamentError('Columns must be a list, '
'%s found' % type(cols))
self._cols = cols
self._tabular = Tabular(self._cols)
self._tabular.padding_width = 10
self._tabular.junction_char = '|'
def add_row(row):
if not isinstance(row, list):
raise FilamentError('Expected list type for the row, found %s'
% type(row))
self._tabular.add_row(row)
def sort_by(col, sort_desc=True):
if len(self._cols) == 0:
raise FilamentError('Expected at least 1 column but 0 found')
if col not in self._cols:
raise FilamentError('%s column does not exist' % col)
self._sort_by = col
self._sort_desc = sort_desc
def limit(l):
if len(self._cols) == 0:
raise FilamentError('Expected at least 1 column but 0 found')
if not type(l) is int:
raise FilamentError('Limit must be an integer value')
self._limit = l
def title(text):
self._tabular.title = text
self._filament_module.columns = columns
self._filament_module.title = title
self._filament_module.sort_by = sort_by
self._filament_module.limit = limit
self._filament_module.add_row = add_row
self._filament_module.render_tabular = self.render_tabular
on_init = self._find_filament_func('on_init')
if on_init and self._zero_args(on_init):
self._filament_module.on_init()
if self._find_filament_func('on_interval'):
self.scheduler.add_executor(ThreadPoolExecutor(max_workers=4))
self.scheduler.start()
def on_interval():
try:
self._filament_module.on_interval()
except Exception:
self._logger.error('Unexpected error on interval elapsed %s'
% traceback.format_exc())
self.scheduler.add_job(on_interval,
IntervalTrigger(),
seconds=self._interval,
max_instances=4,
misfire_grace_time=60)
if len(self._cols) > 0:
try:
self._ansi_term.setup_console()
except TermInitializationError:
panic('fibratus run: ERROR - console initialization failed')
def do_output_accessors(self, outputs):
"""Creates the filament's output accessors.
Parameters
----------
outputs: dict
outputs initialized from the configuration
descriptor
"""
for name, output in outputs.items():
setattr(self._filament_module, name, OutputAccessor(output))
def on_next_kevent(self, kevent):
try:
self._filament_module.on_next_kevent(ddict(kevent))
except Exception as e:
self._logger.error('Unexpected filament error %s' % e)
def render_tabular(self):
"""Renders the table on the console.
"""
if len(self._cols) > 0:
tabular = self._tabular.get_string(start=1, end=self._limit)
if self._sort_by:
tabular = self._tabular.get_string(start=1, end=self._limit,
sortby=self._sort_by,
reversesort=self._sort_desc)
self._tabular.clear_rows()
self._ansi_term.write_output(tabular)
def close(self):
on_stop = self._find_filament_func('on_stop')
if on_stop and self._zero_args(on_stop):
self._filament_module.on_stop()
if self.scheduler.running:
self.scheduler.shutdown()
self._ansi_term.restore_console()
@classmethod
def exists(cls, filament):
Filament._assert_root_dir()
return os.path.exists(os.path.join(FILAMENTS_DIR, '%s.py' % filament))
@classmethod
def list_filaments(cls):
Filament._assert_root_dir()
filaments = {}
paths = [os.path.join(FILAMENTS_DIR, path) for path in os.listdir(FILAMENTS_DIR)
if path.endswith('.py')]
for path in paths:
filament_name = os.path.basename(path)[:-3]
loader = SourceFileLoader(filament_name, path)
filament = loader.load_module()
filaments[filament_name] = inspect.getdoc(filament)
return filaments
@classmethod
def _assert_root_dir(cls):
if not os.path.exists(FILAMENTS_DIR):
panic('fibratus run: ERROR - %s path does not exist.' % FILAMENTS_DIR)
@property
def filters(self):
return self._filters
@property
def logger(self):
return self._logger
@logger.setter
def logger(self, logger):
self._logger = logger
@property
def filament_module(self):
return self._filament_module
@property
def name(self):
return self._name
def _find_filament_func(self, func_name):
"""Finds the function in the filament module.
Parameters
----------
func_name: str
the name of the function
"""
functions = inspect.getmembers(self._filament_module, predicate=inspect.isfunction)
return next(iter([func for name, func in functions if name == func_name]), None)
def _find_filament_path(self, filament_name):
"""Resolves the filament full path from the name
Parameters
----------
filament_name: str
the name of the filament whose path if about to be resolved
"""
return next(iter([os.path.join(FILAMENTS_DIR, filament) for filament in os.listdir(FILAMENTS_DIR)
if filament.endswith('.py') and filament_name == filament[:-3]]), None)
def _num_args(self, func):
return len(inspect.getargspec(func).args)
def _zero_args(self, func):
return self._num_args(func) == 0
class Filament():
"""Filament initialization and execution engine.
Filaments are lightweight Python modules which run
on top of Fibratus. They are often used to enrich/extend the
functionality of Fibratus by performing any type of logic
(aggregations, groupings, filters, counters, etc) on the
kernel event stream.
"""
def __init__(self):
self._filament = None
self._filters = []
self._tabular = None
self._cols = []
self._limit = 10
self._interval = 1
self._sort_by = None
self._sort_desc = True
self.ansi_term = AnsiTerm()
self.scheduler = BackgroundScheduler()
self.term_initialized = False
self._on_stop = None
def load_filament(self, name):
Filament._assert_root_dir()
[filament_path] = [os.path.join(FILAMENTS_DIR, filament) for filament in os.listdir(FILAMENTS_DIR)
if filament.endswith('.py') and name == filament[:-3]] or [None]
if filament_path:
loader = SourceFileLoader(name, filament_path)
self._filament = loader.load_module()
# check for required methods
# on the filament module
doc = inspect.getdoc(self._filament)
if not doc:
raise FilamentError("Please provide a short description for the filament")
[on_next_kevent] = self._find_func('on_next_kevent')
if on_next_kevent:
args_spec = inspect.getargspec(on_next_kevent)
if len(args_spec.args) != 1:
raise FilamentError('Missing one argument on_next_kevent method on filament')
else:
raise FilamentError('Missing required on_next_kevent method on filament')
else:
raise FilamentError('%s filament not found' % name)
def initialize_filament(self):
if self._filament:
def set_filter(*args):
self._filters = args
self._filament.set_filter = set_filter
def set_interval(interval):
if not type(interval) is int:
raise FilamentError('Interval must be an integer value')
self._interval = interval
self._filament.set_interval = set_interval
def columns(cols):
if not isinstance(cols, list):
raise FilamentError('Columns must be a list, %s found' % type(cols))
self._cols = cols
self._tabular = PrettyTable(self._cols)
self._tabular.padding_width = 10
self._tabular.junction_char = '|'
def sort_by(col, sort_desc=True):
if len(self._cols) == 0:
raise FilamentError('Expected at least 1 column but 0 found')
if not col in self._cols:
raise FilamentError('%s column does not exist' % col)
self._sort_by = col
self._sort_desc = sort_desc
def limit(limit):
if len(self._cols) == 0:
raise FilamentError('Expected at least 1 column but 0 found')
if not type(limit) is int:
raise FilamentError('Limit must be an integer value')
self._limit = limit
def title(text):
self._tabular.title = text
def add_row(row):
if not isinstance(row, list):
raise FilamentError('Expected list type for the row')
self._tabular.add_row(row)
self._filament.columns = columns
self._filament.title = title
self._filament.sort_by = sort_by
self._filament.limit = limit
self._filament.add_row = add_row
self._filament.render_tabular = self.render_tabular
# call filaments methods if defined
[on_init] = self._find_func('on_init')
if on_init:
if len(inspect.getargspec(on_init).args) == 0:
self._filament.on_init()
[on_stop] = self._find_func('on_stop')
if on_stop:
if len(inspect.getargspec(on_stop).args) == 0:
self._on_stop = on_stop
[on_interval] = self._find_func('on_interval')
if on_interval:
self.scheduler.add_executor(ThreadPoolExecutor(max_workers=8))
self.scheduler.start()
self.scheduler.add_job(self._filament.on_interval,
'interval',
seconds=1, max_instances=8,
misfire_grace_time=60)
def render_tabular(self):
if len(self._cols) > 0:
tabular = self._tabular.get_string(start=1, end=self._limit)
if self._sort_by:
tabular = self._tabular.get_string(start=1, end=self._limit,
sortby=self._sort_by,
reversesort=self._sort_desc)
if not self.term_initialized:
self.term_initialized = True
self.ansi_term.init_console()
self._tabular.clear_rows()
self.ansi_term.cls()
self.ansi_term.write(tabular)
def process(self, kevent):
self._filament.on_next_kevent(kevent)
def close(self):
if self._on_stop:
self._on_stop()
if self.scheduler.running:
self.scheduler.shutdown()
self.ansi_term.restore_console()
@classmethod
def exists(cls, filament):
Filament._assert_root_dir()
return os.path.exists(os.path.join(FILAMENTS_DIR, '%s.py' % filament))
@classmethod
def list_filaments(cls):
Filament._assert_root_dir()
filaments = {}
paths = [os.path.join(FILAMENTS_DIR, path) for path in os.listdir(FILAMENTS_DIR)
if path.endswith('.py')]
for path in paths:
filament_name = os.path.basename(path)[:-3]
loader = SourceFileLoader(filament_name, path)
filament = loader.load_module()
filaments[filament_name] = inspect.getdoc(filament)
return filaments
@property
def filters(self):
return self._filters
@classmethod
def _assert_root_dir(cls):
if not os.path.exists(FILAMENTS_DIR):
IO.write_console('fibratus run: ERROR - %s path does not exist.' % FILAMENTS_DIR)
sys.exit(0)
def _find_func(self, func_name):
functions = inspect.getmembers(self._filament, predicate=inspect.isfunction)
return [func for name, func in functions if name == func_name] or [None]
import os, time
from datetime import datetime
from apscheduler.schedulers.background import BackgroundScheduler
def myjob_1():
print('myjob_1: %s' % datetime.now())
def myjob_2():
print('myjob_2: %s' % datetime.now())
if __name__ == '__main__':
scheduler = BackgroundScheduler()
scheduler.add_executor('debug')
scheduler.add_executor('threadpool')
# scheduler.add_executor('processpool')
scheduler.add_job(myjob_1, 'interval', seconds=2)
scheduler.add_job(myjob_2, 'interval', seconds=5)
scheduler.start()
jobs = scheduler.get_jobs()
print(jobs)
try:
while True:
time.sleep(5)
except (KeyboardInterrupt, SystemExit):
scheduler.shutdown()
import time
from kafka import KafkaConsumer
from kafka import KafkaProducer
from kafka.errors import KafkaError, KafkaTimeoutError
import tweepy
from tweepy.auth import OAuthHandler
from tweepy.error import TweepError
from flask import Flask, request, jsonify
from apscheduler.schedulers.background import BackgroundScheduler
logging.basicConfig()
logger = logging.getLogger("tweet_producer")
logger.setLevel(logging.DEBUG)
scheduler = BackgroundScheduler()
scheduler.add_executor("threadpool")
scheduler.start()
app = Flask(__name__)
app.config.from_envvar('CONFIG_FILE')
watchlist = set(["aapl"])
kafka_broker = app.config["KAFKA_BROKER"]
consumer_key = app.config["CONSUMER_KEY"]
consumer_secret = app.config["CONSUMER_SECRET"]
access_token = app.config["ACCESS_TOKEN"]
access_secret = app.config["ACCESS_SECRET"]
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
tw_api = tweepy.API(auth)
from apscheduler.jobstores.redis import RedisJobStore
from apscheduler.schedulers.background import BackgroundScheduler
import datetime
import time
from apscheduler.schedulers . blocking import BlockingScheduler
import threading
import decorator_libs
def func(x,y,d):
time.sleep(5)
print(f'{d} {x} + {y} = {x + y}')
scheduler = BackgroundScheduler()
# scheduler.add_job(func,'date', run_date=datetime.datetime(2021, 5, 19, 10, 53, 20), args=(5, 6,))
# print(scheduler._executors)
scheduler.add_executor(ThreadPoolExecutor(1))
# scheduler.add_jobstore(RedisJobStore(jobs_key='apscheduler.jobs', run_times_key='apscheduler.run_times',db=6))
# scheduler.add_jobstore()
def job_miss(event):
# print(event)
# print(event.__dict__)
# print(scheduler.get_job(event.job_id))
print(event.function,event.function_kwargs,event.function_args)
scheduler.add_listener(job_miss, EVENT_JOB_MISSED)
scheduler.start()
with decorator_libs.TimerContextManager():
class Schedule():
def __init__(self, host, port):
self.host =host
self.port = port
self.client = MongoClient(self.host, self.port)
self.sched = BackgroundScheduler()
self.jobstore = MongoDBJobStore(collection='scanner', database='dns_scanner', client=self.client)
self.sched.add_jobstore(self.jobstore)
self.sched.add_executor(ThreadPoolExecutor(10))
self.mongomanager = MongoManager(self.client)
self.sched.start()
self.joblist = []
self.redisquere = RedisQueue("dns_scanner")
logging.info("Schedule init success")
print "init success"
def rm_all_task(self):
self.jobstore.remove_all_jobs()
def rm_task(self, id):
if id in self.joblist:
self.jobstore.remove_job(id)
def run(self):
while True:
try:
cfgstr = self.redisquere.popmsg()
if not cfgstr or not str(cfgstr).startswith("{"):
time.sleep(10)
continue
logging.info(cfgstr)
conf = json.loads(cfgstr)
taskname = conf.get("taskname", "")
taskid = conf.get("taskid", "")
interval = conf.get("interval", {})
seconds = interval.get("seconds", 0)
minutes = interval.get("minutes", 0)
hours = interval.get("hours", 0)
days = interval.get("days", 0)
weeks = interval.get("weeks", 0)
if weeks + days + hours + hours + minutes + seconds < 1:
logging.error(taskname + "-->>interval error")
time.sleep(10)
continue
logging.info(taskname + " add job")
taskinfo = {
"taskname": taskname,
"_id": taskid,
"conf": conf,
"create_time": time.time(),
"times":0,
"process":0
}
if not self.mongomanager.insert(taskinfo):
logging.error(taskname + "-->>in mongo error")
time.sleep(10)
continue
logging.info(taskname + "-->>in mongo success")
job = self.sched.add_job(runjobs, "interval", seconds=seconds, minutes=minutes, hours=hours, days=days,
weeks=weeks, next_run_time=datetime.datetime.now(),args=[taskid,self.host, self.port])
self.joblist.append(job)
time.sleep(70)
except Exception, e:
logging.error(str(e))
class TestSchedulerListener(unittest.TestCase):
def setUp(self):
self.scheduler = BackgroundScheduler()
self.scheduler.add_jobstore(MemoryJobStore(), alias='in_memory')
self.scheduler.add_executor(ThreadPoolExecutor(1), alias='secondary_executor')
self.scheduler.start()
def tearDown(self):
self.scheduler.shutdown()
def test_watcher_injection(self):
watcher = SchedulerWatcher(self.scheduler)
self.assertEqual(watcher.scheduler, self.scheduler, 'Watcher should keep a reference to the scheduler')
self.assertEqual(1, len(self.scheduler._listeners), 'Watcher should inject itself as a scheduler listener')
self.assertEqual(
self.scheduler._listeners[0][1], EVENT_ALL, 'Watcher should register iself to watch all events'
)
def test_scheduler_inspection(self):
self.scheduler.add_job(lambda: 0, jobstore='in_memory', trigger='interval', minutes=60, id='test_job')
watcher = SchedulerWatcher(self.scheduler)
self.assertEqual('running', watcher.scheduler_info['state'], 'Watcher should inspect scheduler status')
self.assertEqual(
str(self.scheduler.timezone),
watcher.scheduler_info['timezone'],
'Watcher should inspect scheduler timezone'
)
self.assertEqual(
'BackgroundScheduler', watcher.scheduler_info['class'], 'Watcher should inspect scheduler class'
)
self.assertEqual(2, len(watcher.jobstores), 'Watcher should inspect all scheduler jobstores')
self.assertIn('in_memory', watcher.jobstores, 'Watcher should have inspected the in_memory jobstore')
self.assertEqual(2, len(watcher.executors), 'Watcher should inspect all scheduler executors')
self.assertIn('secondary_executor', watcher.executors, 'Watcher should have inspected the secondary_executor')
self.assertEqual(1, len(watcher.jobs), 'Watcher should inspect all jobs in scheduler on init')
self.assertIn('test_job', watcher.jobs, 'Watcher should index jobs by id')
def test_job_properties_on_add(self):
watcher = SchedulerWatcher(self.scheduler)
self.scheduler.add_job(
lambda x, y: x + y,
id='added_job',
name='Added job',
jobstore='in_memory',
trigger='interval',
minutes=60,
args=(1,),
kwargs={'y': 2}
)
self.assertIn('added_job', watcher.jobs)
job_properties = watcher.jobs['added_job']['properties']
self.assertEqual('added_job', job_properties['id'], 'Job properties should have the job id')
self.assertEqual('Added job', job_properties['name'], 'Job properties should have the job name')
self.assertIn('trigger', job_properties, 'Job properties should have a representation of the trigger')
self.assertEqual('in_memory', job_properties['jobstore'], 'Job properties should have the jobstore name')
self.assertEqual('default', job_properties['executor'], 'Job properties should have the executor name')
self.assertIn('lambda', job_properties['func'], 'Job properties should have the function string repr')
self.assertIn('func_ref', job_properties, 'Job properties should have the function reference')
self.assertEqual('(1,)', job_properties['args'], 'Job properties should have the job arguments')
self.assertEqual("{'y': 2}", job_properties['kwargs'], 'Job properties should have the job keyword arguments')
self.assertIn('pending', job_properties, 'Job properties should have the job pending status')
self.assertFalse(job_properties['pending'], 'Job status should not be pending')
self.assertIn('coalesce', job_properties, 'Job properties should have the job coalesce configuration')
self.assertIn('next_run_time', job_properties, 'Job properties should have the next run time calculated')
self.assertIn('misfire_grace_time', job_properties, 'Job properties should have the misfire grace time')
self.assertIn('max_instances', job_properties, 'Job properties should have the max instances configuration')
def test_job_inspection_matches_job_added_event(self):
# We're going to add two jobs that should have the exact same properties, except for the id, in two different
# stages of the usage: before the watcher is created and after we start watching for events.
def job_function(x, y):
return x + y
next_run_time = datetime.now() + timedelta(hours=1)
# Job that is added before the user calls us.
self.scheduler.add_job(
job_function,
id='job_1',
name='Added job',
jobstore='in_memory',
trigger='interval',
minutes=60,
args=(1,),
kwargs={'y': 2},
next_run_time=next_run_time
)
watcher = SchedulerWatcher(self.scheduler)
# Job that gets added after we start watching.
self.scheduler.add_job(
job_function,
id='job_2',
name='Added job',
jobstore='in_memory',
trigger='interval',
minutes=60,
args=(1,),
kwargs={'y': 2},
next_run_time=next_run_time
)
self.assertEqual(2, len(watcher.jobs))
job_1 = watcher.jobs['job_1']
job_2 = watcher.jobs['job_2']
for property_name in job_1['properties'].keys():
# All properties, except the id, should match.
if property_name == 'id':
continue
self.assertEqual(job_1['properties'][property_name], job_2['properties'][property_name])
def test_all_events_have_a_processing_method(self):
for event_name in list(SchedulerWatcher.apscheduler_events.values()):
self.assertIn(event_name, dir(SchedulerWatcher))
def test_job_execution_monitoring(self):
watcher = SchedulerWatcher(self.scheduler)
self.scheduler.add_job(
lambda: time.sleep(0.02),
id='waiting_job',
name='Waiting job',
jobstore='in_memory',
trigger='interval',
seconds=0.2,
next_run_time=datetime.now()
)
job_events = watcher.jobs['waiting_job']['events']
self.assertEqual(1, len(job_events))
self.assertEqual('job_added', job_events[0]['event_name'])
time.sleep(0.05)
self.assertEqual(3, len(job_events), 'Job execution needs to be tracked in job events')
self.assertEqual(
'job_submitted',
job_events[1]['event_name'],
'Job submision needs to be tracked in job events'
)
self.assertEqual('job_executed', job_events[2]['event_name'], 'Job execution needs to be tracked in job events')
time.sleep(0.2)
self.assertEqual(5, len(job_events), 'Subsequent executions get tracked')
def test_job_failure_monitoring(self):
watcher = SchedulerWatcher(self.scheduler)
def fail():
time.sleep(0.02)
return 0 / 0
self.scheduler.add_job(
fail,
id='failing_job',
name='Failing job',
jobstore='in_memory',
trigger='interval',
next_run_time=datetime.now(),
minutes=60
)
failing_job_events = watcher.jobs['failing_job']['events']
time.sleep(0.05)
self.assertEqual(3, len(failing_job_events))
self.assertEqual('job_error', failing_job_events[2]['event_name'])
def test_scheduler_summary(self):
watcher = SchedulerWatcher(self.scheduler)
summary = watcher.scheduler_summary()
self.assertEqual(sorted(['scheduler', 'jobs', 'executors', 'jobstores']), sorted(summary.keys()))
self.assertEqual('running', summary['scheduler']['state'], 'scheduler_summary should have the scheduler status')
self.assertEqual(2, len(summary['executors']), 'scheduler_summaru should have the two added executors')
self.assertEqual(2, len(summary['jobstores']), 'scheduler_summary should have the two executors')
self.assertEqual(0, len(summary['jobs']), 'scheduler_summary should have no jobs')
self.scheduler.add_job(lambda: 0, id='job_1')
summary = watcher.scheduler_summary()
self.assertIn('job_1', summary['jobs'], 'scheduler_summary should have the added jobs in it')
self.scheduler.remove_job('job_1')
summary = watcher.scheduler_summary()
self.assertIn('job_1', summary['jobs'], 'scheduler_summary should have all jobs in it, even if job was removed')
def test_removed_jobs_are_only_flagged_as_removed(self):
self.scheduler.add_job(lambda: 0, id='a_job')
watcher = SchedulerWatcher(self.scheduler)
self.assertIn('a_job', watcher.jobs)
self.assertIsNone(watcher.jobs['a_job']['removed_time'])
self.scheduler.remove_job('a_job')
self.assertIn('a_job', watcher.jobs, 'removed jobs should be still tracked in the scheduler watcher')
self.assertIsNotNone(watcher.jobs['a_job']['removed_time'], 'removed_time should be set')
def test_modified_job_properties_are_tracked(self):
self.scheduler.add_job(
lambda x, y: x + y,
id='a_job',
name='A job',
jobstore='in_memory',
trigger='interval',
minutes=60,
args=(1,),
kwargs={'y': 2}
)
watcher = SchedulerWatcher(self.scheduler)
self.assertEqual(watcher.jobs['a_job']['modified_time'], watcher.jobs['a_job']['added_time'])
next_run_time = watcher.jobs['a_job']['properties']['next_run_time'][0]
self.scheduler.modify_job('a_job', name='A modified job', next_run_time=datetime.now() + timedelta(days=1))
self.assertGreater(watcher.jobs['a_job']['modified_time'], watcher.jobs['a_job']['added_time'])
self.assertEqual('A modified job', watcher.jobs['a_job']['properties']['name'])
self.assertGreater(watcher.jobs['a_job']['properties']['next_run_time'][0], next_run_time)
@patch('apschedulerui.watcher.SchedulerWatcher.notify_jobstore_event')
def test_removing_a_jobstore_removes_all_jobs(self, mock_notify_jobstore_event):
watcher = SchedulerWatcher(self.scheduler)
self.scheduler.add_job(lambda: 0, id='job_1', jobstore='in_memory', trigger='interval', minutes=60)
self.scheduler.add_job(lambda: 0, id='job_2', jobstore='in_memory', trigger='interval', minutes=60)
self.assertEqual(2, len(watcher.jobs))
self.assertIsNone(watcher.jobs['job_1']['removed_time'], 'job_1 removed time should be None')
self.assertEqual('in_memory', watcher.jobs['job_1']['properties']['jobstore'])
self.scheduler.remove_jobstore('in_memory')
mock_notify_jobstore_event.assert_called()
self.assertEqual(2, len(watcher.jobs), 'The amount of jobs after removing a jobstore should not change')
self.assertIsNotNone(watcher.jobs['job_1']['removed_time'], 'job_1 removed time should be set')
self.assertIsNotNone(watcher.jobs['job_2']['removed_time'], 'job_2 removed time should be set')
@patch('apschedulerui.watcher.SchedulerWatcher._repr_job')
@patch('apschedulerui.watcher.SchedulerWatcher.notify_job_event')
@patch('apschedulerui.watcher.SchedulerWatcher.notify_jobstore_event')
def test_adding_a_jobstore_adds_all_jobs_in_it(self, mock_notify_jobstore_event, mock_notify_job_event, _):
watcher = SchedulerWatcher(self.scheduler)
jobstore = MemoryJobStore()
jobstore.add_job(Job(scheduler=self.scheduler, id='job_1', next_run_time=datetime.now() + timedelta(days=1)))
jobstore.add_job(Job(scheduler=self.scheduler, id='job_2', next_run_time=datetime.now() + timedelta(days=2)))
self.assertEqual(0, len(watcher.jobs))
self.scheduler.add_jobstore(jobstore, alias='in_memory_2')
self.assertIn('in_memory_2', watcher.jobstores, 'Watcher should have the new jobstore tracked')
self.assertEqual(2, len(watcher.jobs), 'Watcher should add all jobs in the newly added jobstore')
self.assertTrue(all([job_id in watcher.jobs for job_id in ['job_1', 'job_2']]))
self.assertEqual(2, mock_notify_job_event.call_count)
mock_notify_jobstore_event.assert_called_once()
@patch('apschedulerui.watcher.SchedulerWatcher.notify_job_event')
def test_removing_all_jobs_flags_all_as_removed(self, mock_notify_job_event):
watcher = SchedulerWatcher(self.scheduler)
self.scheduler.add_job(lambda: 0, id='job_1', jobstore='default', trigger='interval', minutes=60)
self.scheduler.add_job(lambda: 0, id='job_2', jobstore='in_memory', trigger='interval', minutes=60)
self.assertEqual(2, len(watcher.jobs))
self.assertEqual(2, mock_notify_job_event.call_count)
mock_notify_job_event.reset_mock()
self.scheduler.remove_all_jobs()
self.assertEqual(2, len(watcher.jobs), 'job count should not change after removing all jobs')
self.assertEqual(2, mock_notify_job_event.call_count)
@patch('apschedulerui.watcher.SchedulerWatcher.notify_executor_event')
def test_adding_and_removing_executors(self, mock_notify_executor_event):
watcher = SchedulerWatcher(self.scheduler)
self.scheduler.add_executor(ThreadPoolExecutor(), alias='new_executor')
self.assertIn('new_executor', watcher.executors)
mock_notify_executor_event.assert_called()
mock_notify_executor_event.reset_mock()
self.scheduler.remove_executor('new_executor')
self.assertNotIn('new_executor', watcher.executors)
mock_notify_executor_event.assert_called()
def test_job_event_history_is_limited(self):
watcher = SchedulerWatcher(self.scheduler, max_events_per_job=4)
self.scheduler.add_job(lambda: 0, trigger='interval', seconds=0.01, id='recurrent_job')
time.sleep(0.1)
# recurrent_job should have been executed ~10 times now, generating ~20 events (submission + execution).
self.assertEqual(
watcher.max_events_per_job,
len(watcher.jobs['recurrent_job']['events']),
'job event history should be limited'
)
class Filament():
"""Filament initialization and execution engine.
Filaments are lightweight Python modules which run
on top of Fibratus. They are often used to enrich/extend the
functionality of Fibratus by performing any type of logic
(aggregations, groupings, filters, counters, etc) on the
kernel event stream.
"""
def __init__(self):
self._filament = None
self._filters = []
self._tabular = None
self._cols = []
self._limit = 10
self._interval = 1
self._sort_by = None
self._sort_desc = True
self.ansi_term = AnsiTerm()
self.scheduler = BackgroundScheduler()
self.term_initialized = False
self._on_stop = None
def load_filament(self, name):
Filament._assert_root_dir()
[filament_path] = [
os.path.join(FILAMENTS_DIR, filament)
for filament in os.listdir(FILAMENTS_DIR)
if filament.endswith('.py') and name == filament[:-3]
] or [None]
if filament_path:
loader = SourceFileLoader(name, filament_path)
self._filament = loader.load_module()
# check for required methods
# on the filament module
doc = inspect.getdoc(self._filament)
if not doc:
raise FilamentError(
"Please provide a short description for the filament")
[on_next_kevent] = self._find_func('on_next_kevent')
if on_next_kevent:
args_spec = inspect.getargspec(on_next_kevent)
if len(args_spec.args) != 1:
raise FilamentError(
'Missing one argument on_next_kevent method on filament'
)
else:
raise FilamentError(
'Missing required on_next_kevent method on filament')
else:
raise FilamentError('%s filament not found' % name)
def initialize_filament(self):
if self._filament:
def set_filter(*args):
self._filters = args
self._filament.set_filter = set_filter
def set_interval(interval):
if not type(interval) is int:
raise FilamentError('Interval must be an integer value')
self._interval = interval
self._filament.set_interval = set_interval
def columns(cols):
if not isinstance(cols, list):
raise FilamentError('Columns must be a list, %s found' %
type(cols))
self._cols = cols
self._tabular = PrettyTable(self._cols)
self._tabular.padding_width = 10
self._tabular.junction_char = '|'
def sort_by(col, sort_desc=True):
if len(self._cols) == 0:
raise FilamentError(
'Expected at least 1 column but 0 found')
if not col in self._cols:
raise FilamentError('%s column does not exist' % col)
self._sort_by = col
self._sort_desc = sort_desc
def limit(limit):
if len(self._cols) == 0:
raise FilamentError(
'Expected at least 1 column but 0 found')
if not type(limit) is int:
raise FilamentError('Limit must be an integer value')
self._limit = limit
def title(text):
self._tabular.title = text
def add_row(row):
if not isinstance(row, list):
raise FilamentError('Expected list type for the row')
self._tabular.add_row(row)
self._filament.columns = columns
self._filament.title = title
self._filament.sort_by = sort_by
self._filament.limit = limit
self._filament.add_row = add_row
self._filament.render_tabular = self.render_tabular
# call filaments methods if defined
[on_init] = self._find_func('on_init')
if on_init:
if len(inspect.getargspec(on_init).args) == 0:
self._filament.on_init()
[on_stop] = self._find_func('on_stop')
if on_stop:
if len(inspect.getargspec(on_stop).args) == 0:
self._on_stop = on_stop
[on_interval] = self._find_func('on_interval')
if on_interval:
self.scheduler.add_executor(ThreadPoolExecutor(max_workers=8))
self.scheduler.start()
self.scheduler.add_job(self._filament.on_interval,
'interval',
seconds=1,
max_instances=8,
misfire_grace_time=60)
def render_tabular(self):
if len(self._cols) > 0:
tabular = self._tabular.get_string(start=1, end=self._limit)
if self._sort_by:
tabular = self._tabular.get_string(start=1,
end=self._limit,
sortby=self._sort_by,
reversesort=self._sort_desc)
if not self.term_initialized:
self.term_initialized = True
self.ansi_term.init_console()
self._tabular.clear_rows()
self.ansi_term.cls()
self.ansi_term.write(tabular)
def process(self, kevent):
self._filament.on_next_kevent(kevent)
def close(self):
if self._on_stop:
self._on_stop()
if self.scheduler.running:
self.scheduler.shutdown()
self.ansi_term.restore_console()
@classmethod
def exists(cls, filament):
Filament._assert_root_dir()
return os.path.exists(os.path.join(FILAMENTS_DIR, '%s.py' % filament))
@classmethod
def list_filaments(cls):
Filament._assert_root_dir()
filaments = {}
paths = [
os.path.join(FILAMENTS_DIR, path)
for path in os.listdir(FILAMENTS_DIR) if path.endswith('.py')
]
for path in paths:
filament_name = os.path.basename(path)[:-3]
loader = SourceFileLoader(filament_name, path)
filament = loader.load_module()
filaments[filament_name] = inspect.getdoc(filament)
return filaments
@property
def filters(self):
return self._filters
@classmethod
def _assert_root_dir(cls):
if not os.path.exists(FILAMENTS_DIR):
IO.write_console('fibratus run: ERROR - %s path does not exist.' %
FILAMENTS_DIR)
sys.exit(0)
def _find_func(self, func_name):
functions = inspect.getmembers(self._filament,
predicate=inspect.isfunction)
return [func
for name, func in functions if name == func_name] or [None]
import json
import logging
from apscheduler.events import EVENT_JOB_EXECUTED, EVENT_JOB_ERROR, EVENT_JOB_ADDED
from apscheduler.schedulers.background import BackgroundScheduler
from sqlalchemy.orm.exc import NoResultFound
from .db import engine, metadata, Session
from .models import JobResult
logger = logging.getLogger(__name__)
scheduler = BackgroundScheduler()
scheduler.add_jobstore('sqlalchemy', engine=engine, metadata=metadata)
scheduler.add_executor('processpool')
def on_submit(event):
logger.debug(u'Job [%s] submitted', event.job_id)
db_session = Session()
job_result = JobResult(
job_id=event.job_id,
result=None,
error=0,
status=JobResult.Status.PENDING,
)
db_session.add(job_result)
db_session.commit()
Session.remove()
class TestWebServer(unittest.TestCase):
def setUp(self):
self.scheduler = BackgroundScheduler()
self.scheduler.add_job(lambda: 1,
id='a_job',
trigger='interval',
minutes=10)
self.scheduler.start()
def tearDown(self):
self.scheduler.shutdown()
def test_webserver_init(self):
scheduler_server = SchedulerUI(self.scheduler)
self.assertIsInstance(scheduler_server._scheduler_listener,
SchedulerWatcher)
self.assertRaises(TypeError,
SchedulerUI,
self.scheduler,
operation_timeout=None)
self.assertRaises(ValueError,
SchedulerUI,
self.scheduler,
operation_timeout=-1)
self.assertRaises(TypeError,
SchedulerUI,
self.scheduler,
capabilities=set())
@patch('flask.Flask.add_url_rule')
def test_webserver_capabilities(self, mock_add_url_rule):
SchedulerUI(self.scheduler)
mock_add_url_rule.assert_called()
base_call_count = mock_add_url_rule.call_count
mock_add_url_rule.reset_mock()
SchedulerUI(self.scheduler, capabilities={'pause_job': True})
self.assertEqual(2 + base_call_count, mock_add_url_rule.call_count)
mock_add_url_rule.reset_mock()
SchedulerUI(self.scheduler, capabilities={'run_job': True})
self.assertEqual(1 + base_call_count, mock_add_url_rule.call_count)
mock_add_url_rule.reset_mock()
SchedulerUI(self.scheduler, capabilities={'pause_scheduler': True})
self.assertEqual(
2 + base_call_count, mock_add_url_rule.call_count,
'Web server should register scheduler pause and resume endpoints')
mock_add_url_rule.reset_mock()
SchedulerUI(self.scheduler, capabilities={'stop_scheduler': True})
self.assertEqual(
2 + base_call_count, mock_add_url_rule.call_count,
'Web server should register scheduler stop and start endpoints')
mock_add_url_rule.reset_mock()
SchedulerUI(self.scheduler, capabilities={'remove_job': True})
self.assertEqual(
1 + base_call_count, mock_add_url_rule.call_count,
'Web server should register the endpoint to remove a job')
@patch('flask.Flask.send_static_file')
def test_index_retrieval(self, mock_send_static_file):
SchedulerUI(self.scheduler)._index('/any_path')
mock_send_static_file.assert_called_with('index.html')
@patch('flask.abort')
@patch('apscheduler.schedulers.background.BackgroundScheduler.pause')
def test_scheduler_commands_are_serialized(self, mock_pause, mock_abort):
ui = SchedulerUI(self.scheduler, operation_timeout=0.01)
with ui._scheduler_lock:
# If we acquire the lock, every command we send to the web server should be aborted on lock acquire timeout.
ui._pause_scheduler()
mock_abort.assert_called()
mock_pause.assert_not_called()
ui._resume_scheduler()
ui._stop_scheduler()
ui._start_scheduler()
ui._pause_job('a_job')
ui._resume_job('a_job')
ui._run_job('a_job')
ui._remove_job('a_job')
self.assertEqual(8, mock_abort.call_count)
@patch('apscheduler.schedulers.background.BackgroundScheduler.pause')
@patch('apscheduler.schedulers.background.BackgroundScheduler.resume')
@patch('apscheduler.schedulers.background.BackgroundScheduler.shutdown')
@patch('apscheduler.schedulers.background.BackgroundScheduler.start')
def test_scheduler_requests(self, mock_start, mock_shutdown, mock_resume,
mock_pause):
ui = SchedulerUI(self.scheduler)
ui._pause_scheduler()
mock_pause.assert_called()
ui._resume_scheduler()
mock_resume.assert_called()
ui._stop_scheduler()
mock_shutdown.assert_called()
ui._start_scheduler()
mock_start.assert_called()
@patch('apscheduler.schedulers.background.BackgroundScheduler.remove_job')
@patch('apscheduler.schedulers.background.BackgroundScheduler.resume_job')
@patch('apscheduler.schedulers.background.BackgroundScheduler.pause_job')
def test_job_requests(self, mock_pause_job, mock_resume_job,
mock_remove_job):
ui = SchedulerUI(self.scheduler)
ui._pause_job('a_job')
mock_pause_job.assert_called()
ui._resume_job('a_job')
mock_resume_job.assert_called()
ui._remove_job('a_job')
mock_remove_job.assert_called()
@patch('flask.abort')
def test_missing_jobs_requests_are_aborted(self, mock_abort):
ui = SchedulerUI(self.scheduler)
ui._pause_job('non_existing_job')
ui._resume_job('non_existing_job')
ui._run_job('non_existing_job')
self.assertEqual(3, mock_abort.call_count)
mock_abort.reset_mock()
response = ui._run_job(job_id=None)
self.assertEqual(response.status_code, 404,
'Requests with missing job_id should fail')
@patch('flask_socketio.SocketIO.run')
def test_webserver_start(self, mock_run):
ui = SchedulerUI(self.scheduler)
self.assertEqual(0, len(ui._scheduler_listener.listeners))
ui.start()
self.assertEqual(1, len(ui._scheduler_listener.listeners))
self.assertEqual(ui, ui._scheduler_listener.listeners[0],
'Webserver should register itself as listener')
# SocketIO.run should be called by the web server thread on start.
mock_run.assert_called_with(ui._web_server, host='0.0.0.0', port=5000)
@patch('flask_socketio.SocketIO.emit')
@patch('flask_socketio.SocketIO.run')
def test_scheduler_events_are_emitted_to_clients(self, mock_run,
mock_emit):
ui = SchedulerUI(self.scheduler)
ui.start()
mock_run.assert_called()
# Pause scheduler.
self.scheduler.pause()
mock_emit.assert_called_once()
self.assertEqual('scheduler_paused',
mock_emit.call_args[0][1]['event_name'])
mock_emit.reset_mock()
# Resume it.
self.scheduler.resume()
self.assertEqual('scheduler_resumed',
mock_emit.call_args[0][1]['event_name'])
# Stop it.
mock_emit.reset_mock()
self.scheduler.shutdown()
self.assertEqual('scheduler_shutdown',
mock_emit.call_args[0][1]['event_name'])
# Start it again.
mock_emit.reset_mock()
self.scheduler.start()
self.assertEqual('scheduler_started',
mock_emit.call_args[0][1]['event_name'])
@patch('flask_socketio.SocketIO.emit')
@patch('flask_socketio.SocketIO.run')
def test_job_events_are_emitted_to_clients(self, mock_run, mock_emit):
ui = SchedulerUI(self.scheduler)
ui.start()
self.scheduler.add_job(lambda: time.sleep(0.1),
id='waiting_job',
name='Waiting job',
trigger='interval',
seconds=0.2,
next_run_time=datetime.now() +
timedelta(milliseconds=50))
mock_emit.assert_called_once()
emitted_event = mock_emit.call_args[0][1]
self.assertEqual('job_added', emitted_event['event_name'])
self.assertIn('properties', emitted_event)
self.assertIsInstance(
datetime.strptime(emitted_event['event_ts'],
'%Y-%m-%d %H:%M:%S.%f'), datetime)
mock_emit.reset_mock()
time.sleep(0.1)
# Job submission event.
mock_emit.assert_called_once()
@patch('flask_socketio.SocketIO.emit')
@patch('flask_socketio.SocketIO.run')
def test_jobstore_events_are_emitted_to_clients(self, mock_run, mock_emit):
ui = SchedulerUI(self.scheduler)
ui.start()
# Job store addition.
self.scheduler.add_jobstore(MemoryJobStore(), alias='in_memory')
mock_emit.assert_called_once()
self.assertEqual('jobstore_event', mock_emit.call_args[0][0])
emitted_event = mock_emit.call_args[0][1]
self.assertEqual('in_memory', emitted_event['jobstore_name'])
self.assertEqual('jobstore_added', emitted_event['event_name'])
self.assertIsInstance(
datetime.strptime(emitted_event['event_ts'],
'%Y-%m-%d %H:%M:%S.%f'), datetime)
# Job store removal.
mock_emit.reset_mock()
self.scheduler.remove_jobstore('in_memory')
mock_emit.assert_called_once()
emitted_event = mock_emit.call_args[0][1]
self.assertEqual('in_memory', emitted_event['jobstore_name'])
self.assertEqual('jobstore_removed', emitted_event['event_name'])
self.assertIsInstance(
datetime.strptime(emitted_event['event_ts'],
'%Y-%m-%d %H:%M:%S.%f'), datetime)
@patch('flask_socketio.SocketIO.emit')
@patch('flask_socketio.SocketIO.run')
def test_executors_events_are_emitted_to_clients(self, mock_run,
mock_emit):
ui = SchedulerUI(self.scheduler)
ui.start()
# Executor addition.
self.scheduler.add_executor(ThreadPoolExecutor(max_workers=1),
alias='thread_pool')
mock_emit.assert_called_once()
self.assertEqual('executor_event', mock_emit.call_args[0][0])
emitted_event = mock_emit.call_args[0][1]
self.assertEqual('thread_pool', emitted_event['executor_name'])
self.assertEqual('executor_added', emitted_event['event_name'])
self.assertIsInstance(
datetime.strptime(emitted_event['event_ts'],
'%Y-%m-%d %H:%M:%S.%f'), datetime)
# Executor removal.
mock_emit.reset_mock()
self.scheduler.remove_executor('thread_pool')
mock_emit.assert_called_once()
emitted_event = mock_emit.call_args[0][1]
self.assertEqual('thread_pool', emitted_event['executor_name'])
self.assertEqual('executor_removed', emitted_event['event_name'])
self.assertIsInstance(
datetime.strptime(emitted_event['event_ts'],
'%Y-%m-%d %H:%M:%S.%f'), datetime)
@patch('flask_socketio.emit')
def test_connected_clients_get_initialized(self, mock_emit):
ui = SchedulerUI(self.scheduler, capabilities={'run_job': True})
ui.start(port=5001, host='localhost')
time.sleep(0.1)
import socketio
socket_client = socketio.Client()
socket_client.connect('ws://localhost:5001')
socket_client.emit(
'connected'
) # Notify server that we're now connected, as frontend would do.
time.sleep(0.1)
self.assertEqual(2, mock_emit.call_count,
'emit should be called twice when a client connects')
first_call = mock_emit.call_args_list[0]
second_call = mock_emit.call_args_list[1]
self.assertEqual(
'init_jobs', first_call[0][0],
'First argument of the first emit should be event name')
self.assertEqual(
'init_capabilities', second_call[0][0],
'First argument of the second emit shoud be the init_capabilities event name'
)
self.assertEqual(
ui.capabilities, second_call[0][1],
"Second argument of init_capabilities should equal the web server's capabilities"
)
socket_client.disconnect()
logging.basicConfig()
#import click
app = Flask(__name__)
# Load default config and override config from an environment variable
app.config.update(dict(
DATABASE=os.path.join(app.root_path, 'db', 'notifications.db'),
SECRET_KEY='secret',
USERNAME='******',
PASSWORD='******'
))
# scheduler
sched = BackgroundScheduler(timezone='America/Los_Angeles')
sched.add_jobstore('sqlalchemy', url='sqlite:///db/notifications.db')
sched.add_executor('threadpool')
sched.start()
def connect_db():
"""Connects to the specific database."""
rv = sqlite3.connect(app.config['DATABASE'])
rv.row_factory = dict_factory
return rv
@app.cli.command('initdb')
@click.option('--file',help='the schema file')
def initdb_command(file):
initdb(file)
def initdb(file):
"""Initialize the database."""
finishedMutex.acquire()
finishedThreads = finishedThreads + 1
print "[Worker {0}]\tFinished items(count me): {1}".format(worker, finishedThreads)
if finishedThreads==numWorkers:
print "[Worker {0}]\tAll finished. Send event".format(worker)
event.set()
finishedMutex.release()
print "[Worker {0}]\tEXIT".format(worker)
finishedThreads = 0
jobs = range(1, 51)
worker = 0
mutex = threading.Lock()
finishedMutex = threading.Lock()
event = threading.Event()
scheduler = BackgroundScheduler()
scheduler.add_executor(ThreadPoolExecutor(max_workers=20))
scheduler.start()
for job in jobs:
worker += 1
scheduler.add_job(threadFunc, args=[worker, len(jobs)], max_instances=1000, misfire_grace_time=86400)
print "[Main]\t\tWait for event"
event.wait()
print "[Main]\t\tExit"
from kafka.errors import (KafkaError, KafkaTimeoutError)
logger_format = '%(asctime)-15s %(message)s'
logging.basicConfig(format=logger_format)
logger = logging.getLogger('data-producer')
logger.setLevel(logging.INFO)
app = Flask(__name__)
app.config.from_envvar('ENV_CONFIG_FILE')
kafka_broker = app.config['CONFIG_KAFKA_ENDPOINT']
topic_name = app.config['CONFIG_KAFKA_TOPIC']
producer = KafkaProducer(bootstrap_servers=kafka_broker)
schedule = BackgroundScheduler()
schedule.add_executor('threadpool')
schedule.start()
symbols = set()
def shutdown_hook():
"""
a shutdown hook to be called before the shutdown
"""
try:
logger.info(
'Flushing pending messages to kafka, timeout is set to 10s')
producer.flush(10)
logger.info('Finish flushing pending messages to kafka')
except KafkaError as kafka_error:
def getScheduler():
scheduler = BackgroundScheduler()
scheduler.add_executor("threadpool")
return scheduler
def send_data_to_kafka(producer,topic):
for stock_symbol in stock_names_set:
msg = getQuotes(stock_symbol)
producer.send(topic=topic,value=json.dumps(msg))
print('the stock symbol is %s and the result is %s' % (str(stock_symbol),str(msg)))
def shutdown_hook(producer,schedulers):
producer.flush(10)
producer.close()
schedulers.shutdown()
if __name__=='__main__':
parser = ConfigParser.ConfigParser()
parser.read(os.getcwd() + '/redis_config.ini')
kafka_cluster = parser.get('kafka_config','cluster')
kafka_topic = parser.get('kafka_config','topic')
producer = KafkaProducer(bootstrap_servers=kafka_cluster)
schedulers = BackgroundScheduler()
schedulers.add_executor(ThreadPoolExecutor(5))
schedulers.add_job(send_data_to_kafka, 'interval', [producer, kafka_topic], seconds=3)
schedulers.start()
atexit.register(shutdown_hook, producer,schedulers)
app.run(host='localhost',port=8081)
# init and run scheduler
# 启动调度
scheduler.start()
second_redis_jobstore = RedisJobStore(
db=2,
jobs_key="apschedulers.second_jobs",
run_times_key="apschedulers.second_run_times",
host="127.0.0.1",
port=6379,
password="******"
)
scheduler.add_jobstore(second_redis_jobstore, 'second')
# 定义executor 使用asyncio是的调度执行规则
second_executor = AsyncIOExecutor()
scheduler.add_executor(second_executor, "second")
# *********** 关于 APScheduler中有关Event相关使用示例 *************
# 定义函数监听事件
def job_execute(event):
"""
监听事件处理
:param event:
:return:
"""
print(
"job执行job:\ncode => {}\njob.id => {}\njobstore=>{}".format(
event.code,
event.job_id,
event.jobstore
class Filament(object):
"""Filament initialization and execution engine.
Filaments are lightweight Python modules which run
on top of Fibratus. They are often used to enrich/extend the
functionality of Fibratus by performing any type of logic
(aggregations, groupings, filters, counters, etc) on the
kernel event stream.
"""
def __init__(self):
"""Builds a new instance of the filament.
Attributes:
----------
filament_module: module
module which contains the filament logic
"""
self._filament_module = None
self._name = None
self._filters = []
self._cols = []
self._tabular = None
self._limit = 10
self._interval = 1
self._sort_by = None
self._sort_desc = True
self._logger = None
self._ansi_term = AnsiTerm()
self.scheduler = BackgroundScheduler()
def load_filament(self, name):
"""Loads the filament module.
Finds and loads the python module which
holds the filament logic. It also looks up for
some essential filament methods and raises an error
if they can't be found.
Parameters
----------
name: str
name of the filament to load
"""
self._name = name
Filament._assert_root_dir()
filament_path = self._find_filament_path(name)
if filament_path:
loader = SourceFileLoader(name, filament_path)
self._filament_module = loader.load_module()
sys.path.append(FILAMENTS_DIR)
doc = inspect.getdoc(self._filament_module)
if not doc:
raise FilamentError('Please provide a short '
'description for the filament')
on_next_kevent = self._find_filament_func('on_next_kevent')
if on_next_kevent:
if self._num_args(on_next_kevent) != 1:
raise FilamentError('Missing one argument on_next_kevent '
'method on filament')
self._initialize_funcs()
else:
raise FilamentError('Missing required on_next_kevent '
'method on filament')
else:
raise FilamentError('%s filament not found' % name)
def _initialize_funcs(self):
"""Setup the filament modules functions.
Functions
---------
set_filter: func
accepts the comma separated list of kernel events
for whose the filter should be applied
set_interval: func
establishes the fixed repeating interval in seconds
columns: func
configure the column set for the table
add_row: func
adds a new row to the table
sort_by: func
sorts the table by specific column
"""
def set_filter(*args):
self._filters = args
self._filament_module.set_filter = set_filter
def set_interval(interval):
if not type(interval) is int:
raise FilamentError('Interval must be an integer value')
self._interval = interval
self._filament_module.set_interval = set_interval
def columns(cols):
if not isinstance(cols, list):
raise FilamentError('Columns must be a list, '
'%s found' % type(cols))
self._cols = cols
self._tabular = Tabular(self._cols)
self._tabular.padding_width = 10
self._tabular.junction_char = '|'
def add_row(row):
if not isinstance(row, list):
raise FilamentError(
'Expected list type for the row, found %s' % type(row))
self._tabular.add_row(row)
def sort_by(col, sort_desc=True):
if len(self._cols) == 0:
raise FilamentError('Expected at least 1 column but 0 found')
if col not in self._cols:
raise FilamentError('%s column does not exist' % col)
self._sort_by = col
self._sort_desc = sort_desc
def limit(l):
if len(self._cols) == 0:
raise FilamentError('Expected at least 1 column but 0 found')
if not type(l) is int:
raise FilamentError('Limit must be an integer value')
self._limit = l
def title(text):
self._tabular.title = text
self._filament_module.columns = columns
self._filament_module.title = title
self._filament_module.sort_by = sort_by
self._filament_module.limit = limit
self._filament_module.add_row = add_row
self._filament_module.render_tabular = self.render_tabular
on_init = self._find_filament_func('on_init')
if on_init and self._zero_args(on_init):
self._filament_module.on_init()
if self._find_filament_func('on_interval'):
self.scheduler.add_executor(ThreadPoolExecutor(max_workers=4))
self.scheduler.start()
def on_interval():
try:
self._filament_module.on_interval()
except Exception:
self._logger.error(
'Unexpected error on interval elapsed %s' %
traceback.format_exc())
self.scheduler.add_job(on_interval,
'interval',
seconds=self._interval,
max_instances=4,
misfire_grace_time=60)
if len(self._cols) > 0:
try:
self._ansi_term.setup_console()
except TermInitializationError:
panic('fibratus run: ERROR - console initialization failed')
def do_output_accessors(self, outputs):
"""Creates the filament's output accessors.
Parameters
----------
outputs: dict
outputs initialized from the configuration
descriptor
"""
for name, output in outputs.items():
setattr(self._filament_module, name, OutputAccessor(output))
def on_next_kevent(self, kevent):
try:
self._filament_module.on_next_kevent(ddict(kevent))
except Exception as e:
self._logger.error('Unexpected filament error %s' % e)
def render_tabular(self):
"""Renders the table on the console.
"""
if len(self._cols) > 0:
tabular = self._tabular.get_string(start=1, end=self._limit)
if self._sort_by:
tabular = self._tabular.get_string(start=1,
end=self._limit,
sortby=self._sort_by,
reversesort=self._sort_desc)
self._tabular.clear_rows()
self._ansi_term.write_output(tabular)
def close(self):
on_stop = self._find_filament_func('on_stop')
if on_stop and self._zero_args(on_stop):
self._filament_module.on_stop()
if self.scheduler.running:
self.scheduler.shutdown()
self._ansi_term.restore_console()
@classmethod
def exists(cls, filament):
Filament._assert_root_dir()
return os.path.exists(os.path.join(FILAMENTS_DIR, '%s.py' % filament))
@classmethod
def list_filaments(cls):
Filament._assert_root_dir()
filaments = {}
paths = [
os.path.join(FILAMENTS_DIR, path)
for path in os.listdir(FILAMENTS_DIR) if path.endswith('.py')
]
for path in paths:
filament_name = os.path.basename(path)[:-3]
loader = SourceFileLoader(filament_name, path)
filament = loader.load_module()
filaments[filament_name] = inspect.getdoc(filament)
return filaments
@classmethod
def _assert_root_dir(cls):
if not os.path.exists(FILAMENTS_DIR):
panic('fibratus run: ERROR - %s path does not exist.' %
FILAMENTS_DIR)
@property
def filters(self):
return self._filters
@property
def logger(self):
return self._logger
@logger.setter
def logger(self, logger):
self._logger = logger
@property
def filament_module(self):
return self._filament_module
@property
def name(self):
return self._name
def _find_filament_func(self, func_name):
"""Finds the function in the filament module.
Parameters
----------
func_name: str
the name of the function
"""
functions = inspect.getmembers(self._filament_module,
predicate=inspect.isfunction)
return next(
iter([func for name, func in functions if name == func_name]),
None)
def _find_filament_path(self, filament_name):
"""Resolves the filament full path from the name
Parameters
----------
filament_name: str
the name of the filament whose path if about to be resolved
"""
return next(
iter([
os.path.join(FILAMENTS_DIR, filament)
for filament in os.listdir(FILAMENTS_DIR)
if filament.endswith('.py') and filament_name == filament[:-3]
]), None)
def _num_args(self, func):
return len(inspect.getargspec(func).args)
def _zero_args(self, func):
return self._num_args(func) == 0
return str(tz)
ansible_scheduler = BackgroundScheduler({
'apscheduler.job_defaults.coalesce':
'true',
'apscheduler.job_defaults.max_instances':
'5',
'apscheduler.timezone':
get_timezone(),
})
logger.error("create backgroupd schedule with id=%s" % id(ansible_scheduler))
ansible_scheduler.add_jobstore(DjangoJobStore(), 'default')
# ansible_scheduler.add_executor(ProcessPoolExecutor(10), 'default')
ansible_scheduler.add_executor(ThreadPoolExecutor(20), 'default')
ansible_scheduler.start()
def run_cron_job(op, group, entity):
url = ansible_get_git_url()
prj_name = url.split('/')[-1].split('.')[0]
message = {}
message['event'] = "APSchedule"
message['type'] = "routine"
message["role"] = op
message['group'] = group
message["host"] = entity
message["src_path"] = ANSIBLE_PATH + prj_name
message['username'], message['password'] = get_host_username_and_password(
entity)
