def handle(self, *args, **options):
now = datetime.now()
lock_time = now - settings.FSCAN_TDELTA['max']
orphan_proc = Scan.objects.filter(
status=settings.SCAN_STATUS['in_process'],
last_updated__lte=lock_time)
for obj in orphan_proc:
obj.unlock_task('find_scans call unlock task')
for scan_task in ScanTask.objects.filter(
status=settings.TASK_STATUS['free']):
min_time_allowed = now - settings.FSCAN_TDELTA['max']
# start at fixed time
if scan_task.start:
now_delta = now - scan_task.start
if (now_delta < settings.FSCAN_TDELTA['max'] and
now_delta > settings.FSCAN_TDELTA['min']):
if not Scan.objects.filter(scan_task=scan_task.id,
finish__gte=min_time_allowed):
logger.info('Found waiting time scan: %s' % scan_task.target)
scan_task.run()
# cron-schedule start
if scan_task.cron:
job = CronExpression(smart_str(scan_task.cron))
if job.check_trigger((now.year, now.month, now.day, now.hour,
now.minute)):
if not Scan.objects.filter(scan_task=scan_task.id,
finish__gte=min_time_allowed):
logger.info('Found waiting cron scan: %s' % scan_task.target)
scan_task.run()
def handle(self, *args, **options):
now = datetime.now()
lock_time = now - settings.FSCAN_TDELTA['max']
orphan_proc = Scan.objects.filter(
status=settings.SCAN_STATUS['in_process'],
last_updated__lte=lock_time)
for obj in orphan_proc:
obj.unlock_task('find_scans call unlock task')
for scan_task in ScanTask.objects.filter(
status=settings.TASK_STATUS['free']):
min_time_allowed = now - settings.FSCAN_TDELTA['max']
# start at fixed time
if scan_task.start:
now_delta = now - scan_task.start
if (now_delta < settings.FSCAN_TDELTA['max']
and now_delta > settings.FSCAN_TDELTA['min']):
if not Scan.objects.filter(scan_task=scan_task.id,
finish__gte=min_time_allowed):
logger.info('Found waiting time scan: %s' %
scan_task.target)
scan_task.run()
# cron-schedule start
if scan_task.cron:
job = CronExpression(smart_str(scan_task.cron))
if job.check_trigger(
(now.year, now.month, now.day, now.hour, now.minute)):
if not Scan.objects.filter(scan_task=scan_task.id,
finish__gte=min_time_allowed):
logger.info('Found waiting cron scan: %s' %
scan_task.target)
scan_task.run()
def __init__(
self, interval: Optional[DurationLiteral] = 60, instant_run: bool = False, **kwargs: Any
):
super().__init__(**kwargs)
self._assert_polling_compat()
if interval is None:
# No scheduled execution. Use endpoint `/trigger` of api to execute.
self._poll_interval = None
self.interval = None
self.is_cron = False
else:
try:
# Literals such as 60s, 1m, 1h, ...
self._poll_interval = parse_duration_literal(interval)
self.interval = interval
self.is_cron = False
except TypeError:
# ... or a cron-like expression is valid
from cronex import CronExpression # type: ignore
self._cron_interval = CronExpression(interval)
self.interval = self._cron_interval
self.is_cron = True
self._is_running = False
self._scheduler: Optional[Scheduler] = None
self._instant_run = try_parse_bool(instant_run, False)
def __init__(self, method, cron_expression, custom_messages = {}):
"""
Constructor
@Params
method : Method to be called when the timer is triggered
minutes_between_posts : Time between calls of the method in minutes
custom_messages : Custom log messages for console
"""
self._messages = {
"triggering": "Triggering",
"starting": "Starting",
"syncing": "Syncing",
"exiting": "Exiting"
}
# if only a few messages were chosen, overwrite those ones
# rather than leave the others blank
if custom_messages is not {}:
for message in custom_messages:
self._messages[message] = custom_messages[message]
self._job = CronExpression(cron_expression + " Run timed method")
self._method = method
def __init__(self,
actor_config,
cron="*/10 * * * *",
payload="wishbone",
field="@data"):
Actor.__init__(self, actor_config)
self.pool.createQueue("outbox")
self.cron = CronExpression("%s wishbone" % self.kwargs.cron)
def should_run(schedule, date):
cron = CronExpression(" ".join([
schedule.minute or "*",
schedule.hour or "*",
schedule.day or "*",
schedule.month or "*",
schedule.weekday or "*",
]))
now = [date.year, date.month, date.day, date.hour, date.minute]
return cron.check_trigger(now)
def __init__(self, expressions, **kwargs):
super().__init__(interval=60, instant_run=False, **kwargs)
self.expressions = make_list(expressions)
from cronex import CronExpression
self.jobs = [
CronExpression(expression) for expression in self.expressions
]
class Cron(InputModule):
'''**Generates an event at the defined time**
Generates an event with the defined payload at the chosen time.
Time is in crontab format.
Parameters::
- native_events(bool)(False)
| Whether to expect incoming events to be native Wishbone events
- cron(string)("*/10 * * * *")
| The cron expression.
- payload(str)("wishbone")
| The content of <destination>.
- destination(str)("data")
| The location to write <payload> to.
Queues::
- outbox
| Outgoing messges
'''
def __init__(self, actor_config, native_events=False,
cron="*/10 * * * *", payload="wishbone", destination="data"):
Actor.__init__(self, actor_config)
self.pool.createQueue("outbox")
self.cron = CronExpression("%s wishbone" % self.kwargs.cron)
def preHook(self):
self.sendToBackground(self.timer)
def timer(self):
while self.loop():
if self.cron.check_trigger(time.localtime(time.time())[:5]):
self.logging.info("Cron executed.")
for chunk in [self.kwargs_raw["payload"], None]:
for payload in self.decode(chunk):
event = self.generateEvent(
payload,
self.kwargs.destination
)
self.submit(
event,
"outbox"
)
sleep(60)
def __init__(self,
name,
schedule,
summary_regex=None,
cron_expression=None):
"""
:param name: unique name for this job
:type name: str
:param schedule: the name of the schedule this job runs on
:type schedule: str
:param summary_regex: A regular expression to use for extracting a
string from the job output for use in the summary table. If there is
more than one match, the last one will be used.
:type summary_regex: ``string`` or ``None``
:param cron_expression: A cron-like expression parsable by
`cronex <https://github.com/ericpruitt/cronex>`_ specifying when the
job should run. This has the effect of causing runs to skip this job
unless the expression matches. It's recommended not to use any minute
specifiers and not to use any hour specifiers if the total runtime
of all jobs is more than an hour.
:type cron_expression: str
"""
self._name = name
self._schedule_name = schedule
self._started = False
self._finished = False
self._exit_code = None
self._output = None
self._start_time = None
self._finish_time = None
self._summary_regex = summary_regex
self._skip_reason = None
self._cron_expression = None
if cron_expression is not None:
self._cron_expression = CronExpression(cron_expression)
if not self._cron_expression.check_trigger(
time.gmtime(time.time())[:5]):
self._skip_reason = 'cronex: "%s"' % cron_expression
def validate_schedule_attribute(self, key, attribute):
cron_expression = "{minute} {hour} {day} {month} {weekday}".format(
minute=attribute if key == "minute" else "*",
hour=attribute if key == "hour" else "*",
day=attribute if key == "day" else "*",
month=attribute if key == "month" else "*",
weekday=attribute if key == "weekday" else "*",
)
try:
CronExpression(cron_expression)
except ValueError:
raise ValueError(f"The Schedule cannot accept the "
f"value given in the {key} attribute")
return attribute
class Cron(TimeRepresentation):
"""
Class for representing cron times. This class is meant to be used when we can be certain that on certain days
or in certain times, the transaction should be executing.
Note:
The package we use for cron parsing (cronex) supports using repeaters (for periodicity).
To use repeater put '%number' in expression.
"""
def __init__(self, expression):
"""
Args:
expression (str) : classic cron expression string
"""
self.cronjob = CronExpression(expression)
def evaluate(self, datetime):
self.last_time = datetime
return self.cronjob.check_trigger(datetime.timetuple()[:5])
def evaluate_expression(self):
return self.cronjob.numerical_tab
def get_expression(self):
return self.cronjob.string_tab
def get_epoch(self):
return self.cronjob.epoch
def set_epoch(self, epoch):
"""
Setting the epoch will probably be required for repeaters to work correctly. The epoch is important since
it is the start from which the repeaters will measure the period.
"""
try: # test if the specified parameters are right in terms of a correct datetime
datetime(*epoch[:5])
except:
raise ValueError(
"Some value in the tuple doesn't match a correct date. Use the format: year, month,day,hour,minute."
)
if len(epoch) == 6:
self.cronjob.epoch = epoch
else:
raise ValueError(
"Use tuple with 6 values being in this order: year,month,day,hour,minute,utc_offset!"
)
class Cron(Actor):
'''**Generates an event at the defined time**
Generates an event with the defined payload at the chosen time.
Time is in crontab format.
Parameters:
- cron(string)("*/10 * * * *")
| The cron expression.
- payload(str)("wishbone")
| The content of <field>.
- field(str)("@data")
| The location to write <payload> to.
Queues:
- outbox
| Outgoing messges
'''
def __init__(self,
actor_config,
cron="*/10 * * * *",
payload="wishbone",
field="@data"):
Actor.__init__(self, actor_config)
self.pool.createQueue("outbox")
self.cron = CronExpression("%s wishbone" % self.kwargs.cron)
def preHook(self):
self.sendToBackground(self.timer)
def timer(self):
while self.loop():
if self.cron.check_trigger(time.localtime(time.time())[:5]):
self.logging.info("Cron fired.")
e = Event()
e.set(self.kwargs.payload, self.kwargs.field)
self.submit(e, self.pool.queue.outbox)
sleep(60)
class Cron(Actor):
'''**Generates an event at the defined time**
Generates an event with the defined payload at the chosen time.
Time is in crontab format.
Parameters:
- cron(string)("*/10 * * * *")
| The cron expression.
- payload(str)("wishbone")
| The content of <field>.
- field(str)("@data")
| The location to write <payload> to.
Queues:
- outbox
| Outgoing messges
'''
def __init__(self, actor_config, cron="*/10 * * * *", payload="wishbone", field="@data"):
Actor.__init__(self, actor_config)
self.pool.createQueue("outbox")
self.cron = CronExpression("%s wishbone" % self.kwargs.cron)
def preHook(self):
self.sendToBackground(self.timer)
def timer(self):
while self.loop():
if self.cron.check_trigger(time.localtime(time.time())[:5]):
self.logging.info("Cron fired.")
e = Event()
e.set(self.kwargs.payload, self.kwargs.field)
self.submit(e, self.pool.queue.outbox)
sleep(60)
def __init__(self, expression):
"""
Args:
expression (str) : classic cron expression string
"""
self.cronjob = CronExpression(expression)
def browse_config(config, cache):
"""Will browse all section of the config,
fill cache and launch command when needed.
"""
commands.ip_neigh.cache_clear()
processes = {}
now = datetime.now()
now = (now.year, now.month, now.day, now.hour, now.minute)
excluded_sections = {config.default_section, const.KNOWN_MACHINES_SECTION}
known_machines = utils.get_known_machines(config)
for section in list(config.values()):
if section.name in excluded_sections:
continue
if not section.getboolean('enabled'):
logger.debug('section %r not enabled', section)
continue
cron = section.get('cron')
if cron and not CronExpression(cron).check_trigger(now):
logger.debug('section %r disabled for now', section)
continue
logger.debug('%r - processing section', section.name)
cache.section_name = section.name
device = section.get('device')
neighbors = commands.ip_neigh(device=device)
threshold = section.getint('threshold')
if check_neighborhood(neighbors,
section.get('filter_on_regex'),
section.get('filter_out_regex'),
section.get('filter_on_machines'),
section.get('filter_out_machines'),
section.get('exclude'),
known_machines=known_machines):
cmd = section.get('command_neighbor')
result = 'neighbor'
else:
cmd = section.get('command_no_neighbor')
result = 'no_neighbor'
cache.cache_result(result, threshold)
logger.info('%r - cache state: %r', section.name, cache.section)
count = cache.get_result_count(result)
if count != threshold:
logger.info(
"%r - cache count hasn't reached threshold yet "
"(%d/%d)", section.name, count, threshold)
continue
if cache.last_command == cmd:
logger.info('%r - command has already been run', section.name)
continue
cache.cache_command(cmd)
if cmd:
logger.warning('%r - launching: %r', section.name, cmd)
processes[section.name] = commands.execute(cmd.split())
else:
logger.info('%r - no command to launch', section.name)
handle_processes(processes, config, cache)
def __init__(self, actor_config, cron="*/10 * * * *", payload="wishbone", field="@data"):
Actor.__init__(self, actor_config)
self.pool.createQueue("outbox")
self.cron = CronExpression("%s wishbone" % self.kwargs.cron)
def __init__(self, actor_config, native_events=False,
cron="*/10 * * * *", payload="wishbone", destination="data"):
Actor.__init__(self, actor_config)
self.pool.createQueue("outbox")
self.cron = CronExpression("%s wishbone" % self.kwargs.cron)
class Polling(AsyncPull, AsyncPullNowMixin):
"""
Base class for polling plugins.
You may specify duration literals such as 60 (60 secs), 1m, 1h (...) to realize a periodic
polling or cron expressions (*/1 * * * * > every min) to realize cron like behaviour.
"""
__REPR_FIELDS__ = ['interval', 'is_cron']
def __init__(
self, interval: Optional[DurationLiteral] = 60, instant_run: bool = False, **kwargs: Any
):
super().__init__(**kwargs)
self._assert_polling_compat()
if interval is None:
# No scheduled execution. Use endpoint `/trigger` of api to execute.
self._poll_interval = None
self.interval = None
self.is_cron = False
else:
try:
# Literals such as 60s, 1m, 1h, ...
self._poll_interval = parse_duration_literal(interval)
self.interval = interval
self.is_cron = False
except TypeError:
# ... or a cron-like expression is valid
from cronex import CronExpression # type: ignore
self._cron_interval = CronExpression(interval)
self.interval = self._cron_interval
self.is_cron = True
self._is_running = False
self._scheduler: Optional[Scheduler] = None
self._instant_run = try_parse_bool(instant_run, False)
def _assert_polling_compat(self) -> None:
self._assert_abstract_compat((SyncPolling, AsyncPolling))
self._assert_fun_compat('_poll')
async def _pull(self) -> None:
def _callback() -> None:
loop = asyncio.get_event_loop()
if loop.is_running():
asyncio.ensure_future(self._run_schedule())
self._scheduler = Scheduler()
self._configure_scheduler(self._scheduler, _callback)
if self._instant_run:
self._scheduler.run_all()
while not self.stopped:
self._scheduler.run_pending()
await self._sleep(0.5)
while self._is_running: # Keep the loop alive until the job is finished
await asyncio.sleep(0.1)
async def _pull_now(self) -> None:
await self._run_now()
async def _run_now(self) -> Payload:
"""Runs the poll right now. It will not run, if the last poll is still running."""
if self._is_running:
self.logger.warning("Polling job is still running. Skipping current run")
return
self._is_running = True
try:
payload = await self.poll()
if payload is not None:
self.notify(payload)
return payload
finally:
self._is_running = False
async def _run_schedule(self) -> None:
try:
if self.is_cron:
dtime = datetime.now()
if not self._cron_interval.check_trigger((
dtime.year, dtime.month, dtime.day,
dtime.hour, dtime.minute
)):
return # It is not the time for the cron to trigger
await self._run_now()
except StopPollingError:
await self._stop()
except Exception: # pragma: no cover, pylint: disable=broad-except
self.logger.exception("Polling of '%s' failed", self.name)
def _configure_scheduler(self, scheduler: Scheduler, callback: Callable[[], None]) -> None:
"""
Configures the scheduler. You have to differ between "normal" intervals and
cron like expressions by checking `self.is_cron`.
Override in subclasses to fir the behaviour to your needs.
Args:
scheduler (schedule.Scheduler): The actual scheduler.
callback (callable): The callback to call when the time is right.
Returns:
None
"""
if self.is_cron:
# Scheduler always executes at the exact minute to check for cron triggering
scheduler.every().minute.at(":00").do(callback)
else:
# Only activate when an interval is specified
# If not the only way is to trigger the poll by the api `trigger` endpoint
if self._poll_interval:
# Scheduler executes every interval seconds to execute the poll
scheduler.every(self._poll_interval).seconds.do(callback)
async def poll(self) -> Payload:
"""Performs polling."""
poll_fun = getattr(self, '_poll')
if inspect.iscoroutinefunction(poll_fun):
return await poll_fun()
return await run_sync(poll_fun)
class TimedTrigger:
"""
Calls a method (with no arguments) on a timer
"""
_job = None
_method = None
_messages = {}
def __init__(self, method, cron_expression, custom_messages = {}):
"""
Constructor
@Params
method : Method to be called when the timer is triggered
minutes_between_posts : Time between calls of the method in minutes
custom_messages : Custom log messages for console
"""
self._messages = {
"triggering": "Triggering",
"starting": "Starting",
"syncing": "Syncing",
"exiting": "Exiting"
}
# if only a few messages were chosen, overwrite those ones
# rather than leave the others blank
if custom_messages is not {}:
for message in custom_messages:
self._messages[message] = custom_messages[message]
self._job = CronExpression(cron_expression + " Run timed method")
self._method = method
def start(self):
"""
Begins the timer and starts triggering the method
"""
self._log(self._messages["starting"])
while True:
try:
# if we are not triggering, wait a minute so
# we dont cause the CPU to commit seppuku
#
# for the first pass we wait until the start
# of a new minute anyway
self._wait_until_next_minute()
# check if we are in a minute to trigger on
if self._job.check_trigger(gmtime(time())[:5]):
# run the method
self._run_method()
# if there is a keyboard interrupt during this,
# stop the loop please
except KeyboardInterrupt:
break
self._log(self._messages["exiting"])
def _log(self, message):
"""
Displays a message with the current time as a timestamp
@Params
message : Message to display in console
"""
print("\r[%s] %s" % (str(datetime.now()), message))
def _run_method(self):
"""
Creates a thread for the method and calls it.
Threading is used to call the method. This is done in case
the method takes significant time to complete
"""
thread = Thread(target=self._method)
self._log(self._messages["triggering"])
thread.start()
def _wait_until_next_minute(self):
"""
Waits until the start of a minute.
"""
time_to_wait = 60-gmtime(time())[5]
sleep(time_to_wait)
class Job(object):
"""
Base class for all Job types/classes.
"""
__metaclass__ = abc.ABCMeta
#: Dictionary describing the configuration file schema, to be validated
#: with `jsonschema <https://github.com/Julian/jsonschema>`_.
_schema_dict = {
'type': 'object',
'title': 'Configuration for base Job class',
'properties': {
'name': {
'type': 'string'
},
'schedule': {
'type': 'string'
},
'class_name': {
'type': 'string'
},
'summary_regex': {
'type': 'string'
},
'cron_expression': {
'type': 'string'
}
},
'required': ['name', 'schedule', 'class_name']
}
def __init__(self,
name,
schedule,
summary_regex=None,
cron_expression=None):
"""
:param name: unique name for this job
:type name: str
:param schedule: the name of the schedule this job runs on
:type schedule: str
:param summary_regex: A regular expression to use for extracting a
string from the job output for use in the summary table. If there is
more than one match, the last one will be used.
:type summary_regex: ``string`` or ``None``
:param cron_expression: A cron-like expression parsable by
`cronex <https://github.com/ericpruitt/cronex>`_ specifying when the
job should run. This has the effect of causing runs to skip this job
unless the expression matches. It's recommended not to use any minute
specifiers and not to use any hour specifiers if the total runtime
of all jobs is more than an hour.
:type cron_expression: str
"""
self._name = name
self._schedule_name = schedule
self._started = False
self._finished = False
self._exit_code = None
self._output = None
self._start_time = None
self._finish_time = None
self._summary_regex = summary_regex
self._skip_reason = None
self._cron_expression = None
if cron_expression is not None:
self._cron_expression = CronExpression(cron_expression)
if not self._cron_expression.check_trigger(
time.gmtime(time.time())[:5]):
self._skip_reason = 'cronex: "%s"' % cron_expression
def __repr__(self):
return '<%s name="%s">' % (type(self).__name__, self.name)
@property
def error_repr(self):
"""
Return a detailed representation of the job state for use in error
reporting.
:return: detailed representation of job in case of error
:rtype: str
"""
ecode = ''
if self._exit_code is not None:
ecode = 'Exit Code: %s\n' % self._exit_code
return "%s\nSchedule Name: %s\nStarted: %s\nFinished: %s\n" \
"Duration: %s\n%sOutput: %s\n" % (
self.__repr__(), self._schedule_name, self._started,
self._finished, self.duration, ecode, self._output
)
@property
def name(self):
"""
Return the Job Name.
:return: Job name
:rtype: str
"""
return self._name
@property
def skip(self):
"""
Either None if the job should not be skipped, or a string reason
describing why the Job should be skipped.
:rtype: ``None`` or ``str``
"""
return self._skip_reason
@property
def schedule_name(self):
"""
Return the configured schedule name for this job.
:return: schedule name
:rtype: str
"""
return self._schedule_name
@property
def is_started(self):
"""
Return whether or not the Job has been started.
:return: whether or not the Job has been started
:rtype: bool
"""
return self._started
@property
def is_finished(self):
"""
Return whether or not the Job is finished.
:return: whether or not the Job is finished
:rtype: bool
"""
return self._finished
@property
def exitcode(self):
"""
For Job subclasses that result in a command exit code, return the
integer exitcode. For Job subclasses that result in a boolean (success /
failure) status, return 0 on success or 1 on failure. Returns -1 if the
Job has not completed.
:return: Job exit code or (0 / 1) status
:rtype: int
"""
return self._exit_code
@property
def output(self):
"""
Return the output of the Job as a string, or None if the job has not
completed.
:return: Job output
:rtype: str
"""
return self._output
def summary(self):
"""
Retrieve a simple one-line summary of the Job output/status.
:return: Job one-line summary.
:rtype: str
"""
if self.output is None:
return ''
if self._summary_regex is not None:
res = re.findall(self._summary_regex, self.output, re.M)
if len(res) > 0:
return res[-1]
lines = [x for x in self.output.split("\n") if x.strip() != '']
if len(lines) < 1:
return ''
return lines[-1]
@abc.abstractmethod
def report_description(self):
"""
Return a one-line description of the Job for use in reports.
:rtype: str
"""
raise NotImplementedError()
@property
def duration(self):
"""
Return the duration/runtime of the job, or None if the job did not run.
:return: job duration
:rtype: ``datetime.timedelta`` or ``None``
"""
if self._start_time is None or self._finish_time is None:
return None
return self._finish_time - self._start_time
@abc.abstractmethod
def run(self):
"""
Run the job.
This method sets ``self._started`` and ``self._start_time``. If the Job
runs synchronously, this method also sets ``self._finished``,
``self._exit_code``, ``self._finish_time`` and ``self._output``.
In the case of an exception, this method must still set those attributes
as appropriate and then raise the exception.
:return: True if job finished successfully, False if job finished but
failed, or None if the job is still running in the background.
"""
raise NotImplementedError(
'ERROR: Job subclass must implement run() method.')
def poll(self):
"""
For asynchronous jobs (:py:attr:`~.is_started` is True but
:py:attr:`~.is_finished` is False), check if the job has finished yet.
If not, return ``False``. If the job has finished, update
``self._finish_time``, ``self._exit_code``, ``self._output`` and
``self._finished`` and then return ``True``.
This method should **never** raise exceptions; recoverable exceptions
should be handled via internal retry logic on subsequent poll attempts.
Retries should be done on the next call of this method; we never want
to sleep during this method. Unrecoverable exceptions should set
``self._exit_code``, ``self._output`` and ``self._finished``.
:return: :py:attr:`~.is_finished`
:rtype: bool
"""
return self.is_finished