def test_15_slices(self):
"""Invalid Slices"""
mon = CronSlices('* * * * *')
with self.assertRaises(ValueError):
CronSlices('* * * */15 *')
with self.assertRaises(AssertionError):
mon.setall(mon)
def test_15_slices(self):
"""Invalid Slices"""
mon = CronSlices('* * * * *')
with self.assertRaises(ValueError):
CronSlices('* * * */15 *')
with self.assertRaises(AssertionError):
mon.setall(mon)
def check_argements(handler, pk=None):
error = {}
local_log_file_id = handler.get_argument('local_log_file_id', '')
search_pattern = handler.get_argument('search_pattern', '')
comment = handler.get_argument('comment', '')
alert = handler.get_argument('alert', '2')
crontab_cycle = handler.get_argument('crontab_cycle', '')
check_interval = handler.get_argument('check_interval', '0')
trigger_format = handler.get_argument('trigger_format', '')
dingding_webhook = handler.get_argument('dingding_webhook', '')
if not local_log_file_id:
error['local_log_file_id'] = '日志id是必填项'
if not search_pattern:
error['search_pattern'] = '匹配模式是必填项'
else:
try:
re.search(r'%s' % search_pattern, '')
except:
error['search_pattern'] = '不正确的正则表达式'
else:
select_sql = 'SELECT id FROM local_log_monitor_item WHERE search_pattern="%s" and local_log_file_id="%s" %s' % \
(search_pattern, local_log_file_id, 'and id!="%d"' % pk if pk else '')
count = handler.mysqldb_cursor.execute(select_sql)
if count:
error['search_pattern'] = '匹配模式已存在'
if not comment:
error['comment'] = '备注为必填项'
if alert != '1' and alert != '2':
error['alert'] = '告警选择不正确'
elif alert == '1':
if not crontab_cycle:
error['crontab_cycle'] = '检查周期是必填项'
elif not CronSlices.is_valid(crontab_cycle):
error['crontab_cycle'] = '格式不正确'
if not check_interval:
error['check_interval'] = '检查间隔是必填项'
elif not check_interval.isnumeric():
error['check_interval'] = '必须为正整数'
if not trigger_format:
error['trigger_format'] = '触发公式是必填项'
if not dingding_webhook:
error['dingding_webhook'] = '钉钉webhook是必填项'
data = {
'local_log_file_id': local_log_file_id,
'search_pattern': search_pattern,
'comment': comment,
'alert': alert or '2',
'crontab_cycle': crontab_cycle,
'check_interval': check_interval or '0',
'trigger_format': trigger_format,
'dingding_webhook': dingding_webhook
}
return error, data
def cronHandler(data):
if not data:
return
s = data.split(':')
cronCommand = s[1].strip()
if cronCommand not in ["on", "off"]:
print("Cron command invalid, should be 'on' or 'off'. Command: %s" % cronCommand)
return
cronString = s[2].strip()
if not CronSlices.is_valid(cronString):
print("Cron time string invalid. Time string: %s" % cronString)
return
# Now we've validated the command, set a cron job
cron_file = CronTab(user=True)
it = cron_file.find_command(cronCommand)
try:
job = it.next()
print("Found existing cron task for %s" % cronCommand)
except StopIteration:
job = cron_file.new(command="echo %s >> /tmp/test.txt" % cronCommand)
print("Creating new cron task for %s" % cronCommand)
job.setall(cronString)
job.enable()
cron_file.write()
def get_schedule_interval_in_seconds(self, seconds_ago=0):
"""Return the job interval in seconds or None if there is no interval
:params seconds_ago: return an interval the job had in the past
"""
schedule = self.get_schedule()
if schedule is None:
return None
if CronSlices.is_valid(schedule):
try:
job_tz = pytz.timezone(self.get_schedule_time_zone())
except (pytz.exceptions.UnknownTimeZoneError, AttributeError):
job_tz = pytz.utc
c = croniter(
schedule,
datetime.datetime.now(job_tz) -
datetime.timedelta(seconds=seconds_ago))
return c.get_next() - c.get_prev()
else:
try:
_, _, interval = self.get_schedule().split('/')
return int(
float(isodate.parse_duration(interval).total_seconds()))
except (isodate.ISO8601Error, ValueError):
return None
def __post_init__(self):
if self.hardware.strategy == DistributedStrategy.INFERENCE and not self.server:
raise DescriptorError("Missing server definition")
if self.hardware.strategy in [DistributedStrategy.HOROVOD] and not self.hardware.distributed:
raise DescriptorError("Missing distributed hardware definition")
if self.hardware.distributed:
if self.hardware.distributed.num_instances <= 1:
logging.warning(
f"Specified a distributed strategy but using {self.hardware.distributed.num_instances} nodes"
)
raise DescriptorError(f"Invalid number of instances {self.hardware.distributed.num_instances}")
if self.info.scheduling != SINGLE_RUN_SCHEDULING:
if not CronSlices.is_valid(self.info.scheduling):
raise DescriptorError(
f"Invalid cron expression in scheduling field: {self.info.scheduling}. "
'Please use Kubernetes cron job syntax or "single_run" for non-periodic runs'
)
for label, value in self.info.labels.items():
if not LABEL_VALIDATION_REGEX.fullmatch(label):
raise DescriptorError(f"Invalid custom label key: {label}. " + INVALID_LABEL_MESSAGE)
if value and not LABEL_VALIDATION_REGEX.fullmatch(value):
raise DescriptorError(f"Invalid value for label {label}: {value} " + INVALID_LABEL_MESSAGE)
if self.ml:
if self.ml.framework_version and self.ml.framework == MLFramework.NONE:
raise DescriptorError("Framework version is present, but not framework")
if self.ml.script and not self.ml.script.script.endswith(".tar"):
raise DescriptorError(
f"Script mode section is present, but script file: {self.ml.script.script} is not a tar file"
)
def adjust_schedule(schedule, operation):
logger = applogger("Scheduler")
fields = schedule.split(" ")
M = fields[0]
H = fields[1]
DoM = fields[2]
Mn = fields[3]
DoW = fields[4]
if operation == "increment":
if H == "*":
H = "*"
elif int(H) == 23:
H = 0
else:
H = int(H) + 1
elif operation == "decrement":
if H == "*":
H = "*"
elif int(H) == 0:
H = 23
else:
H = int(H) - 1
elif operation == "None":
logger.error("The provided operation is None")
return
fields[1] = str(H)
parts = " ".join(fields)
if CronSlices.is_valid(parts):
return parts
else:
logger.error("invalid cron expression")
return
def _add_event(self, period_string, event_id):
"""
Add a single event in the crontab.
Will add a line like:
<period_string> python /path/to/kalliope.py start --brain-file /path/to/brain.yml --run-synapse "<event_id>"
E.g:
30 7 * * * python /home/me/kalliope/kalliope.py start --brain-file /home/me/brain.yml --run-synapse "Say-hello"
:param period_string: crontab period
:type period_string: str
:param event_id:
:type event_id: str
:return:
"""
my_user_cron = CronTab(user=True)
job = my_user_cron.new(command=self.base_command + " " +
str("\"" + event_id + "\""),
comment=CRONTAB_COMMENT)
if CronSlices.is_valid(period_string):
job.setall(period_string)
job.enable()
else:
raise InvalidCrontabPeriod("The crontab period %s is not valid" %
period_string)
# write the file
my_user_cron.write()
Utils.print_info("Synapse \"%s\" added to the crontab" % event_id)
def addTaggedCronjob(tag, interval, cmd):
"""
Adds a tagged cronjob to current user's crontab
:param tag: tag for new entry
:type tag: str|int
:param interval: crontab interval
:type interval: str
:param cmd: crontab cmd to run
:type cmd: str
:return: whether it succeeded
:rtype: bool
"""
try:
if isinstance(tag, int):
tag = str(tag)
if not CronSlices.is_valid(interval):
return False
cron = CronTab(user=True)
matching_jobs = tuple(cron.find_comment(tag))
if len(matching_jobs) == 0:
job = cron.new(command=cmd, comment=tag)
else:
job = matching_jobs[0]
job.set_command(cmd)
job.setall(interval)
if not job.is_valid():
return False
cron.write()
return True
except:
return False
def main(env):
here = os.path.dirname(__file__)
config_path = os.path.normpath(here + '/../config.json')
root_path = os.path.normpath(here + '/../')
output = ""
error = ""
# Get POST request data
post_env = env.copy()
post_env['QUERY_STRING'] = ''
post_data = cgi.FieldStorage(fp=env['wsgi.input'],
environ=post_env,
keep_blank_values=True)
# If this is request from configuration form data
if "config" in post_data:
# Prevent saving invalid code.
try:
config = json.loads(post_data['config'].value)
except ValueError:
error += '<center style="color:red;">Config format is invalid!</center>'
else:
# Check and update crontab tasks
for cron_task_name, cron_task in config["cron"].items():
cron = CronTab(user=cron_task["cron_user"])
command = "python3 " + root_path + "/frontend/cron.py " + cron_task_name + " >/dev/null 2>&1"
# Remove old similar jobs, if present
old_jobs = cron.find_command(command)
for old_job in old_jobs:
cron.remove(old_job)
# Create new job
job = cron.new(command=command)
job.setall(cron_task["cron_definition"])
if cron_task["active"]:
job.enable()
else:
job.enable(False)
# Verify and save cron changes
if job.is_valid() and CronSlices.is_valid(
cron_task["cron_definition"]):
cron.write()
else:
error += '<center style="color:red;">Cron definition format is invalid (' + cron_task_name + ')!</center>'
# Write incoming POST data to config.json
if not error:
with open(config_path, 'w') as f:
f.write(post_data["config"].value)
output += '<center style="color:green;">Config saved!</center>'
# Show textaria with contents of config.py
with open(config_path, 'r') as f:
output += error + '<center><h2> Chatwork Bot config editor</h2><form method="post"><div>Please be careful while editing <input style="width:100px;" type="submit" /><div><textarea style="width:500px; height:450px;" name="config">' + f.read(
) + '</textarea></center></form>'
return (output)
def main(env):
here = os.path.dirname(__file__)
config_path = os.path.normpath(here+'/../config.json')
root_path = os.path.normpath(here+'/../')
output = ""
error = ""
# Get POST request data
post_env = env.copy()
post_env['QUERY_STRING'] = ''
post_data = cgi.FieldStorage(
fp=env['wsgi.input'],
environ=post_env,
keep_blank_values=True
)
# If this is request from configuration form data
if "config" in post_data:
# Prevent saving invalid code.
try:
config = json.loads(post_data['config'].value)
except ValueError:
error += '<center style="color:red;">Config format is invalid!</center>'
else:
# Check and update crontab tasks
for cron_task_name, cron_task in config["cron"].items():
cron = CronTab(user=cron_task["cron_user"])
command = "python3 " + root_path + "/frontend/cron.py " + cron_task_name + " >/dev/null 2>&1"
# Remove old similar jobs, if present
old_jobs = cron.find_command(command)
for old_job in old_jobs:
cron.remove(old_job)
# Create new job
job = cron.new(command=command)
job.setall(cron_task["cron_definition"])
if cron_task["active"]:
job.enable()
else:
job.enable(False)
# Verify and save cron changes
if job.is_valid() and CronSlices.is_valid(cron_task["cron_definition"]):
cron.write()
else:
error += '<center style="color:red;">Cron definition format is invalid (' + cron_task_name + ')!</center>'
# Write incoming POST data to config.json
if not error:
with open(config_path, 'w') as f:
f.write(post_data["config"].value)
output += '<center style="color:green;">Config saved!</center>'
# Show textaria with contents of config.py
with open(config_path, 'r') as f:
output += error + '<center><h2> Chatwork Bot config editor</h2><form method="post"><div>Please be careful while editing <input style="width:100px;" type="submit" /><div><textarea style="width:500px; height:450px;" name="config">' + f.read() + '</textarea></center></form>'
return(output)
def cron_validator(value):
"""
校验是否是有效的Cron表达式
:param value:
:return:
"""
if not CronSlices.is_valid(value):
raise ValidationError('不是有效的Crontab表达式')
def crontab(l, r, action):
"""Allow the user to manage crontab entries for the compranet tracker.
The -l option lists the compranet tracker crontab entries and -r removes
them. Two actions are supported, ADD and REMOVE.
\b
To ADD a crontab entry use the following syntax:
compranet-cli crontab add [time] -- [command]
where the time argument is a CRON expression (e.g. "0 0 * * 0" or weekly)
and command is the compranet-cli command to execute.
Example:
compranet-cli crontab add "0 2 * * *" -- "--email-log pull_xlsx"
\b
To REMOVE a crontab entries use the following syntax:
compranet-cli crontab remove [command]
All crontab entries which contain the command argument will be removed.
Example:
compranet-cli crontab remove pull_xlsx
"""
cron = CronTab(user=True)
if l:
for job in cron.find_comment('compranet_tracker'):
print(job)
if r:
for job in cron.find_comment('compranet_tracker'):
cron.remove(job)
cron.write()
if len(action) == 0 and not l and not r:
print(click.get_current_context().get_help())
if len(action) > 0:
if action[0].upper() == 'ADD':
venv_path = settings.VIRTUALENV_PATH
cli_path = os.path.join(venv_path, 'bin', 'compranet-cli')
if len(action) != 3:
raise click.BadParameter("Wrong number of arguments",
param_hint='ACTION')
time = action[1]
if not CronSlices.is_valid(time):
raise click.BadParameter("Invalid CRON expression",
param_hint='ACTION')
cli_opts = ' '.join(action[2:])
command = ' '.join([cli_path, cli_opts])
job = cron.new(command=command, comment='compranet_tracker')
job.setall(time)
cron.write()
elif action[0].upper() == 'REMOVE':
if len(action) != 2:
raise click.BadParameter("Wrong number of arguments",
param_hint='ACTION')
cmd = ' '.join(action[1:])
for job in cron.find_command(cmd):
print("Removing entry {}".format(job))
cron.remove(job)
cron.write()
else:
raise click.BadParameter("Unrecognized action argument",
param_hint='ACTION')
def check_schedule(self):
msgs = []
schedule = self.get_schedule()
if schedule is not None:
if not CronSlices.is_valid(schedule):
try:
repeat, start_time, interval = schedule.split(
'/') # the parts have separate validators
except ValueError:
return (False, (
'The specified schedule "%s" is neither a valid cron schedule nor a valid'
' ISO 8601 schedule' % schedule))
# an empty start time is not valid ISO8601 but Chronos accepts it: '' == current time
if start_time == '':
msgs.append(
'The specified schedule "%s" does not contain a start time'
% schedule)
else:
# Check if start time contains time zone information
try:
dt = isodate.parse_datetime(start_time)
if not hasattr(dt, 'tzinfo'):
msgs.append(
'The specified start time "%s" must contain a time zone'
% start_time)
except isodate.ISO8601Error as exc:
msgs.append(
'The specified start time "%s" in schedule "%s" does '
'not conform to the ISO 8601 format:\n%s' %
(start_time, schedule, exc.message))
parsed_interval = None
try:
# 'interval' and 'duration' are interchangeable terms
parsed_interval = isodate.parse_duration(interval)
except isodate.ISO8601Error:
msgs.append('The specified interval "%s" in schedule "%s" '
'does not conform to the ISO 8601 format.' %
(interval, schedule))
# until we make this configurable, throw an
# error if we have a schedule < 60 seconds (the default schedule_horizone for chronos)
# https://github.com/mesos/chronos/issues/508
if parsed_interval and parsed_interval < datetime.timedelta(
seconds=60):
msgs.append(
'Unsupported interval "%s": jobs must be run at an interval of > 60 seconds'
% interval)
if not self._check_schedule_repeat_helper(repeat):
msgs.append('The specified repeat "%s" in schedule "%s" '
'does not conform to the ISO 8601 format.' %
(repeat, schedule))
return len(msgs) == 0, '\n'.join(msgs)
def check_schedule(self):
msgs = []
schedule = self.get_schedule()
if schedule is not None:
if not CronSlices.is_valid(schedule):
try:
repeat, start_time, interval = schedule.split('/') # the parts have separate validators
except ValueError:
return (
False, (
'The specified schedule "%s" is neither a valid cron schedule nor a valid'
' ISO 8601 schedule' % schedule
),
)
# an empty start time is not valid ISO8601 but Chronos accepts it: '' == current time
if start_time == '':
msgs.append('The specified schedule "%s" does not contain a start time' % schedule)
else:
# Check if start time contains time zone information
try:
dt = isodate.parse_datetime(start_time)
if not hasattr(dt, 'tzinfo'):
msgs.append('The specified start time "%s" must contain a time zone' % start_time)
except isodate.ISO8601Error as exc:
msgs.append('The specified start time "%s" in schedule "%s" does '
'not conform to the ISO 8601 format:\n%s' % (start_time, schedule, str(exc)))
parsed_interval = None
try:
# 'interval' and 'duration' are interchangeable terms
parsed_interval = isodate.parse_duration(interval)
except isodate.ISO8601Error:
msgs.append('The specified interval "%s" in schedule "%s" '
'does not conform to the ISO 8601 format.' % (interval, schedule))
# don't allow schedules more frequent than every minute we have
# to be careful here, since the isodate library returns
# different datatypes according to whether there is a
# yearly/monthly period (and if that year or month period is
# 0). unfortunately, the isodate library *is* okay with you
# specifying fractional and negative periods. Chronos's parser
# will barf at a fractional period, but be okay with a negative
# one, so if someone does try to do something like "R1//P0.01M"
# then the API request to upload the job will fail. TODO:
# detect when someone is trying to add a fractional period?
if(parsed_interval and isinstance(parsed_interval, datetime.timedelta)
and parsed_interval < datetime.timedelta(seconds=60)):
msgs.append('Unsupported interval "%s": jobs must be run at an interval of > 60 seconds' % interval)
if not self._check_schedule_repeat_helper(repeat):
msgs.append('The specified repeat "%s" in schedule "%s" '
'does not conform to the ISO 8601 format.' % (repeat, schedule))
return len(msgs) == 0, '\n'.join(msgs)
def check_schedule(self):
msgs = []
schedule = self.get_schedule()
if schedule is not None:
if not CronSlices.is_valid(schedule):
try:
repeat, start_time, interval = schedule.split("/") # the parts have separate validators
except ValueError:
return (
False,
(
'The specified schedule "%s" is neither a valid cron schedule nor a valid'
" ISO 8601 schedule" % schedule
),
)
# an empty start time is not valid ISO8601 but Chronos accepts it: '' == current time
if start_time == "":
msgs.append('The specified schedule "%s" does not contain a start time' % schedule)
else:
# Check if start time contains time zone information
try:
dt = isodate.parse_datetime(start_time)
if not hasattr(dt, "tzinfo"):
msgs.append('The specified start time "%s" must contain a time zone' % start_time)
except isodate.ISO8601Error as exc:
msgs.append(
'The specified start time "%s" in schedule "%s" does '
"not conform to the ISO 8601 format:\n%s" % (start_time, schedule, exc.message)
)
parsed_interval = None
try:
# 'interval' and 'duration' are interchangeable terms
parsed_interval = isodate.parse_duration(interval)
except isodate.ISO8601Error:
msgs.append(
'The specified interval "%s" in schedule "%s" '
"does not conform to the ISO 8601 format." % (interval, schedule)
)
# until we make this configurable, throw an
# error if we have a schedule < 60 seconds (the default schedule_horizone for chronos)
# https://github.com/mesos/chronos/issues/508
if parsed_interval and parsed_interval < datetime.timedelta(seconds=60):
msgs.append('Unsupported interval "%s": jobs must be run at an interval of > 60 seconds' % interval)
if not self._check_schedule_repeat_helper(repeat):
msgs.append(
'The specified repeat "%s" in schedule "%s" '
"does not conform to the ISO 8601 format." % (repeat, schedule)
)
return len(msgs) == 0, "\n".join(msgs)
def _add_event(self, period_string, event_id):
my_user_cron = CronTab(user=True)
job = my_user_cron.new(command=self.base_command+" "+str("\"" + event_id + "\""), comment=CRONTAB_COMMENT)
if CronSlices.is_valid(period_string):
job.setall(period_string)
job.enable()
else:
raise InvalidCrontabPeriod("The crontab period %s is not valid" % period_string)
# write the file
my_user_cron.write()
Utils.print_info("Synapse \"%s\" added to the crontab" % event_id)
def handle(self, *args, **options):
file_path = options.get('dispatch_file_path')
if os.path.exists(file_path):
with open(file_path) as f:
dispatch = f.read()
if CronSlices.is_valid(dispatch):
set_cron_mail_report(dispatch)
print('Successfully set the scheduled sending of mail Report.')
else:
print('Not a valid cron expression.')
else:
print('The file does not exist.')
def validate_cron(expression):
import sys
# By default the module is installed in the 2.7 branch, pyotherside uses python 3
# We use a custom location
sys.path.append("/usr/share/harbour-sailcron/python/python-crontab")
# https://pypi.python.org/pypi/python-crontab
from crontab import CronSlices
# bool = CronSlices.is_valid('0/2 * * * *')
isValid = CronSlices.is_valid(expression)
return isValid
def main():
module = AnsibleModule(
argument_spec=dict(
# default value for user to bypass required_one_of check
# incase of validate_cron_time
user=dict(default="dpal"),
tabfile=dict(),
use_regex=dict(default=False),
match_string=dict(),
schedule=dict(),
list_all_crons=dict(type=bool),
get_crons_by_command=dict(type=bool),
get_crons_by_comment=dict(type=bool),
get_crons_by_time=dict(type=bool),
validate_cron_time=dict(type=bool),
),
required_one_of=(("user", "tabfile"), ),
required_if=(
("get_crons_by_command", True, ["match_string", "use_regex"]),
("get_crons_by_comment", True, ["match_string", "use_regex"]),
("get_crons_by_time", True, ["match_string", "use_regex"]),
("validate_cron_time", True, ["schedule"]),
))
cron = CronTab(user=module.params["user"],
tabfile=module.params["tabfile"])
if module.params['list_all_crons']:
crons = cron.lines
elif module.params['get_crons_by_command']:
if module.params["use_regex"]:
crons = cron.find_command(
re.compile(r"{}".format(module.params["match_string"])))
else:
crons = cron.find_command(module.params["match_string"])
elif module.params['get_crons_by_comment']:
if module.params["use_regex"]:
crons = cron.find_comment(
re.compile(r"{}".format(module.params["match_string"])))
else:
crons = cron.find_comment(module.params["match_string"])
elif module.params['get_crons_by_time']:
crons = cron.find_time(module.params["match_string"])
elif module.params['validate_cron_time']:
module.exit_json(valid=CronSlices.is_valid(module.params["schedule"]))
else:
module.fail_json(msg="unknown parameters")
module.exit_json(crons=cron_items_to_list(crons))
def validate_cron(expression):
"""Check validity of a cron expression"""
import sys
# By default the module is installed in the 2.7 branch, pyotherside uses python 3
# We use a custom location
sys.path.append("/usr/share/harbour-sailcron/python/python-crontab")
# https://pypi.python.org/pypi/python-crontab
from crontab import CronSlices
# bool = CronSlices.is_valid('0/2 * * * *')
isValid = CronSlices.is_valid(expression)
return isValid
def updateTaggedCronjob(tag, interval='', cmd='', new_tag=''):
"""
Update a tagged cronjob in the current user's crontab
:param tag: tag of existing entry
:type tag: str|int
:param interval: new crontab interval
:type interval: str
:param cmd: new crontab cmd to run
:type cmd: str
:param new_tag: new tag for entry
:type new_tag: str|int
:return: whether it succeeded
:rtype: bool
"""
try:
if isinstance(tag, int):
tag = str(tag)
if isinstance(new_tag, int):
new_tag = str(new_tag)
cron = CronTab(user=True)
matching_jobs = tuple(cron.find_comment(tag))
if len(matching_jobs) == 0:
job = cron.new(comment=tag)
else:
job = tuple(cron.find_comment(tag))[0]
if len(interval) > 0:
if not CronSlices.is_valid(interval):
return False
job.setall(interval)
if len(cmd) > 0:
job.set_command(cmd)
if len(new_tag) > 0:
job.set_comment(new_tag)
if not job.is_valid():
return False
cron.write()
return True
except:
return False
def generate_cron_notation_from_datetime(self,
execution_time=0,
execution_period=24):
"""
When we are creating a cronjob we can set the schedule this way.
:param execution_time:
:param execution_period:
:return:
"""
cron_notation = self.cron_notation
if execution_time == 0 and execution_period != 24:
cron_notation = cron_notation.format(
execution_time='*/', execution_period=execution_period)
if execution_period == 24:
cron_notation = cron_notation.format(execution_time=execution_time,
execution_period='')
if CronSlices.is_valid(cron_notation):
return cron_notation
raise CronNotationDoesNotValidExceptions(e)
def handle(self, directive: str, data: List[Dict[str, Any]]) -> bool:
if directive != "crontab":
self._log.error(f"Can't handle directive {directive}")
return False
cron = CronTab(user=True)
# Remove all existing dotbot crontabs.
updated = cron.remove_all(comment="dotbot") > 0
# Add from config.
for i, entry in enumerate(data):
if "time" not in entry:
self._log.error(f"Skipping entry {i} - missing `time` config")
continue
time = entry.pop("time")
if "command" not in entry:
self._log.error(
f"Skipping entry {i} - missing `command` config")
continue
command = entry.pop("command")
job = cron.new(command=command, comment="dotbot")
if not CronSlices.is_valid(time):
self._log.error(f"Skipping entry {i} - invalid time {time}")
continue
job.setall(time)
if "platform" in entry and entry.pop("platform") != sys.platform:
job.enable(False)
if entry:
self._log.error(f"Unused config keys: {list(entry.keys())}")
updated = True
if updated:
cron.write()
return True
def email_report(request):
error_msg = str()
dispatch = str()
if request.method == 'POST':
dispatch = request.POST.get('dispatch')
if CronSlices.is_valid(dispatch):
set_cron_mail_report(dispatch)
dispatch = str()
error_msg = '设置成功'
else:
error_msg = '不是有效的Crontab表达式'
emails = get_user_emails()
cron = open_crontab()
jobs = list()
for job in cron.find_comment(get_mail_report_cron_comment()):
jobs.append(job.__str__())
return render(
request, 'page/email_report_setting.html', {
'error_msg': error_msg,
'emails': emails,
'crontab': '\n'.join(jobs),
'dispatch': dispatch,
'time': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
})
elif args.mode == "schedule":
DEFAULT_CRON = "0 * * * *" # every hour
if CONFIG_BACKUP_ENABLED is None or CONFIG_BACKUP_ENABLED.lower(
) == "false" or CONFIG_BACKUP_ENABLED.lower() == "off":
log.info("Configuration Backup is not activated.")
sys.exit()
if not os.path.exists(CONFIG_BACKUP_FOLDER):
os.makedirs(CONFIG_BACKUP_FOLDER)
from crontab import CronTab, CronSlices
cron_schedule = DEFAULT_CRON
# env variable can also be a cron scheadule
if CronSlices.is_valid(CONFIG_BACKUP_ENABLED):
cron_schedule = CONFIG_BACKUP_ENABLED
# Cron does not provide enviornment variables, source them manually
environment_file = os.path.join(RESOURCE_FOLDER, "environment.sh")
with open(environment_file, 'w') as fp:
for env in os.environ:
if env != "LS_COLORS":
fp.write("export " + env + "=\"" + os.environ[env] + "\"\n")
os.chmod(environment_file, 0o777)
script_file_path = os.path.realpath(__file__)
command = ". " + environment_file + "; " + sys.executable + " '" + script_file_path + "' backup> /proc/1/fd/1 2>/proc/1/fd/2"
cron = CronTab(user=True)
def valid_cron(cls, value):
if not CronSlices.is_valid(value):
raise ValueError(f"Invalid cron expression: '{value}'")
return value
def validate_cronjob_times(self):
if not CronSlices.is_valid(self.cron_time):
raise ValidationError("Time values are not valid for a cronjob.")
# Write CronTab to new filename::
cron.write( 'output.tab' )
# Write to this user's crontab (unix only)::
cron.write_to_user( user=True )
# Write to some other user's crontab::
cron.write_to_user( user='******' )
# Validate a cron time string::
from crontab import CronSlices
bool = CronSlices.is_valid('0/2 * * * *')
#Environment Variables
#=====================
'''
Some versions of vixie cron support variables outside of the command line.
Sometimes just update the envronment when commands are run, the Cronie fork
of vixie cron also supports CRON_TZ which looks like a regular variable but
actually changes the times the jobs are run at.
Very old vixie crons don't support per-job variables, but most do.
'''
#Iterate through cron level environment variables::
for (name, value) in cron.env.items():
time.sleep(5)
log.info("Start xfce4-panel again.")
log.info("xfce4-panel started with exit code: " + str(subprocess.call("xfce4-panel", shell=True)))
elif args.mode == "schedule":
DEFAULT_CRON = "0 * * * *" # every hour
from crontab import CronTab, CronSlices
cron_schedule = DEFAULT_CRON
script_file_path = os.path.realpath(__file__)
command = sys.executable + " '" + script_file_path + "' check> /proc/1/fd/1 2>/proc/1/fd/2"
cron = CronTab(user=True)
# remove all other tasks
cron.remove_all(command=command)
job = cron.new(command=command)
if CronSlices.is_valid(cron_schedule):
log.info("Scheduling cron check xfdesktop task with with cron: " + cron_schedule)
job.setall(cron_schedule)
job.enable()
cron.write()
else:
log.info("Failed to schedule check xfdesktop. Cron is not valid.")
log.info("Running cron jobs:")
for job in cron:
log.info(job)
def valid_cron(message):
return CronSlices.is_valid(message.content) and message.author.id == ctx.author.id
def _is_valid_crontab(crontab):
if CronSlices.is_valid(crontab):
return crontab
raise argparse.ArgumentTypeError('Invalid crontab.', )
elif args.mode == "schedule":
DEFAULT_CRON = "0 * * * *" # every hour
if WORKSPACE_CONFIG_BACKUP is None or WORKSPACE_CONFIG_BACKUP.lower(
) == "false" or WORKSPACE_CONFIG_BACKUP.lower() == "off":
log.info("Configuration Backup is not activated.")
sys.exit()
if not os.path.exists(WORKSPACE_CONFIG_BACKUP_FOLDER):
os.makedirs(WORKSPACE_CONFIG_BACKUP_FOLDER)
from crontab import CronTab, CronSlices
cron_schedule = DEFAULT_CRON
# env variable can also be a cron scheadule
if CronSlices.is_valid(WORKSPACE_CONFIG_BACKUP):
cron_schedule = WORKSPACE_CONFIG_BACKUP
# Cron does not provide enviornment variables, source them manually
environment_file = os.path.join(RESOURCE_FOLDER, "environment.sh")
with open(environment_file, 'w') as fp:
for env in os.environ:
if env != "LS_COLORS":
fp.write("export " + env + "=\"" + os.environ[env] + "\"\n")
os.chmod(environment_file, 0o777)
script_file_path = os.path.realpath(__file__)
command = ". " + environment_file + "; " + sys.executable + " '" + script_file_path + "' backup> /proc/1/fd/1 2>/proc/1/fd/2"
cron = CronTab(user=True)
def schedule(add: bool=False,
show: bool=False,
clear: bool=False,
foreground: bool=False,
run_all: bool=False,
quiet: bool=False,
every: Optional[str]=None,
depth: int=0,
overwrite: bool=False,
import_path: Optional[str]=None,
out_dir: Path=OUTPUT_DIR):
"""Set ArchiveBox to regularly import URLs at specific times using cron"""
check_data_folder(out_dir=out_dir)
Path(LOGS_DIR).mkdir(exist_ok=True)
cron = CronTab(user=True)
cron = dedupe_cron_jobs(cron)
if clear:
print(cron.remove_all(comment=CRON_COMMENT))
cron.write()
raise SystemExit(0)
existing_jobs = list(cron.find_comment(CRON_COMMENT))
if every or add:
every = every or 'day'
quoted = lambda s: f'"{s}"' if (s and ' ' in str(s)) else str(s)
cmd = [
'cd',
quoted(out_dir),
'&&',
quoted(ARCHIVEBOX_BINARY),
*([
'add',
*(['--overwrite'] if overwrite else []),
f'--depth={depth}',
f'"{import_path}"',
] if import_path else ['update']),
'>>',
quoted(Path(LOGS_DIR) / 'schedule.log'),
'2>&1',
]
new_job = cron.new(command=' '.join(cmd), comment=CRON_COMMENT)
if every in ('minute', 'hour', 'day', 'month', 'year'):
set_every = getattr(new_job.every(), every)
set_every()
elif CronSlices.is_valid(every):
new_job.setall(every)
else:
stderr('{red}[X] Got invalid timeperiod for cron task.{reset}'.format(**ANSI))
stderr(' It must be one of minute/hour/day/month')
stderr(' or a quoted cron-format schedule like:')
stderr(' archivebox init --every=day --depth=1 https://example.com/some/rss/feed.xml')
stderr(' archivebox init --every="0/5 * * * *" --depth=1 https://example.com/some/rss/feed.xml')
raise SystemExit(1)
cron = dedupe_cron_jobs(cron)
cron.write()
total_runs = sum(j.frequency_per_year() for j in cron)
existing_jobs = list(cron.find_comment(CRON_COMMENT))
print()
print('{green}[√] Scheduled new ArchiveBox cron job for user: {} ({} jobs are active).{reset}'.format(USER, len(existing_jobs), **ANSI))
print('\n'.join(f' > {cmd}' if str(cmd) == str(new_job) else f' {cmd}' for cmd in existing_jobs))
if total_runs > 60 and not quiet:
stderr()
stderr('{lightyellow}[!] With the current cron config, ArchiveBox is estimated to run >{} times per year.{reset}'.format(total_runs, **ANSI))
stderr(' Congrats on being an enthusiastic internet archiver! 👌')
stderr()
stderr(' Make sure you have enough storage space available to hold all the data.')
stderr(' Using a compressed/deduped filesystem like ZFS is recommended if you plan on archiving a lot.')
stderr('')
elif show:
if existing_jobs:
print('\n'.join(str(cmd) for cmd in existing_jobs))
else:
stderr('{red}[X] There are no ArchiveBox cron jobs scheduled for your user ({}).{reset}'.format(USER, **ANSI))
stderr(' To schedule a new job, run:')
stderr(' archivebox schedule --every=[timeperiod] --depth=1 https://example.com/some/rss/feed.xml')
raise SystemExit(0)
cron = CronTab(user=True)
cron = dedupe_cron_jobs(cron)
existing_jobs = list(cron.find_comment(CRON_COMMENT))
if foreground or run_all:
if not existing_jobs:
stderr('{red}[X] You must schedule some jobs first before running in foreground mode.{reset}'.format(**ANSI))
stderr(' archivebox schedule --every=hour --depth=1 https://example.com/some/rss/feed.xml')
raise SystemExit(1)
print('{green}[*] Running {} ArchiveBox jobs in foreground task scheduler...{reset}'.format(len(existing_jobs), **ANSI))
if run_all:
try:
for job in existing_jobs:
sys.stdout.write(f' > {job.command.split("/archivebox ")[0].split(" && ")[0]}\n')
sys.stdout.write(f' > {job.command.split("/archivebox ")[-1].split(" >> ")[0]}')
sys.stdout.flush()
job.run()
sys.stdout.write(f'\r √ {job.command.split("/archivebox ")[-1]}\n')
except KeyboardInterrupt:
print('\n{green}[√] Stopped.{reset}'.format(**ANSI))
raise SystemExit(1)
if foreground:
try:
for job in existing_jobs:
print(f' > {job.command.split("/archivebox ")[-1].split(" >> ")[0]}')
for result in cron.run_scheduler():
print(result)
except KeyboardInterrupt:
print('\n{green}[√] Stopped.{reset}'.format(**ANSI))
raise SystemExit(1)
def on_put(self, request, response):
"""
Create a new job
"""
spider_name = request.get_param('spider_name', True)
schedule = request.get_param('schedule')
parameters = request.params
try:
del parameters['spider_name']
del parameters['schedule']
except KeyError:
pass
# Check parameters
if set(parameters.keys()
) & SpiderJobResource.JOB_PARAMETER_RESERVED_KEYWORDS:
response.body = json.dumps({
'status':
'error',
'message':
'Parameter must not be in reserved keywords {}'.format(
SpiderJobResource.JOB_PARAMETER_RESERVED_KEYWORDS)
})
response.status = falcon.HTTP_200
return
# Check spider
try:
spider = Spider.get(Spider.name == spider_name)
except Spider.DoesNotExist:
response.body = json.dumps({
'status':
'error',
'message':
'Cannot find spider by this name'
})
response.status = falcon.HTTP_200
return
# Check schedule
if not CronSlices.is_valid(schedule):
response.body = json.dumps({
'status': 'error',
'message': 'Invalid schedule format'
})
response.status = falcon.HTTP_200
return
try:
job = create_job(spider_name, schedule, parameters)
except Exception as e:
response.body = json.dumps({
'status':
'error',
'message':
'Cannot create job: {}'.format(str(e))
})
response.status = falcon.HTTP_200
return
try:
run_job(job)
except Exception as e:
response.body = json.dumps({
'status':
'error',
'message':
'Cannot run job: {}'.format(str(e)),
'traceback':
traceback.format_exc()
})
response.status = falcon.HTTP_200
return
response.body = json.dumps({
'status': 'scheduled',
'job_id': job.job_id
})
response.status = falcon.HTTP_200
