def check_cluster_cron(cluster_name):
"""
Returns False if the cluster is scaled down according to configured variables. Otherwise True.
:param cluster_name: (str) The name of the cluster to use
:return: Whether or not we should process the cluster
"""
# Process as normal if the cluster isn't in the cluster list
if cluster_name not in SCALE_DOWN_CLUSTERS:
return True
# If the cron expressions are invalid process the cluster
if not croniter.is_valid(SCALE_DOWN_CRON) or not croniter.is_valid(
SCALE_UP_CRON):
logger.warning(
f"Cron expression invalid. Processing {cluster_name} as normal.")
return True
# If the cluster scaled down then don't process the cluster
now = datetime.now()
last_scale_down = croniter(SCALE_DOWN_CRON, now).get_prev(datetime)
last_scale_up = croniter(SCALE_UP_CRON, now).get_prev(datetime)
if last_scale_down > last_scale_up:
logger.info(f"Cluster {cluster_name} is scaled down. Not processing.")
return False
logger.info(f"Cluster {cluster_name} is scaled up. Processing as normal.")
return True
def setup_periodic_tasks(sender, **kwargs):
configurations = ConfigurationLoader().load_by_config_folder()
for configuration in configurations:
with Connection(configuration) as connection:
if connection.has_orm():
connection.orm.tables.create_tables()
if connection.has_mongodb():
connection.mongodb.migrations()
with open(
Path.var_folder_path() + '/' + configuration.hash + '.pickle',
'wb') as handle:
pickle.dump(configuration,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
for configuration_key, aggregationModule in configuration.aggregations.config.items(
):
module = aggregationModule.module
cron = aggregationModule.cron
sender.autodiscover_tasks(['modules.aggregation.custom'], module)
if croniter.is_valid(cron) is True:
(minute, hour, day_month, month, day_week) = str.split(cron,
sep=' ')
sender.add_periodic_task(
crontab(minute, hour, day_week, day_month, month),
run, [
configuration.hash, configuration_key, module,
'modules.aggregation.custom'
],
time_limit=aggregationModule.runtime_limit,
name='aggregation_' + configuration_key)
for configuration_key, operationModule in configuration.operations.config.items(
):
module = operationModule.module
cron = operationModule.cron
sender.autodiscover_tasks(['modules.operation.custom'], module)
if croniter.is_valid(cron) is True:
(minute, hour, day_month, month, day_week) = str.split(cron,
sep=' ')
sender.add_periodic_task(
crontab(minute, hour, day_week, day_month, month),
run, [
configuration.hash, configuration_key, module,
'modules.operation.custom'
],
time_limit=operationModule.runtime_limit,
name='operation_' + configuration_key)
def do_crondtc(self, author, channel, serv, message):
"""Cron feature for DTC printing, max 1 minute per loop with no end
start : lanch cron for DTC, if is not
stop : cancel cron for DTC, if is start
next : print time for next cron DTC loop
update <expression> : change cron DTC frequency, if <expression> is valid"""
items = message.split()
if channel in self._cronDTC:
if message == 'start':
if self._cronDTC[channel].is_alive():
serv.privmsg(channel, 'cronDTC : already start !')
self.__prompt('host', '{%alert%}cronDTC : already start !{%default%}')
else:
serv.privmsg(channel, 'cronDTC : start !')
self.__prompt('host', '{%alert%}cronDTC : start !{%default%}')
elif message == 'stop':
if self._cronDTC[channel].is_alive():
serv.privmsg(channel, 'cronDTC : stopped !')
self.__prompt('host', '{%alert%}cronDTC : stopped !{%default%}')
else:
serv.privmsg(channel, 'cronDTC : already stopped !')
self.__prompt('host', '{%alert%}cronDTC : already stopped !{%default%}')
elif items[0] == 'update':
items.pop(0)
if len(items) > 5:
serv.privmsg(channel, 'cronDTC : update refuse ! (possible abuse frequency)')
self.__prompt('host', '{%alert%}cronDTC : update refuse ! (possible abuse frequency){%default%}')
elif croniter.is_valid(' '.join(items[:5])):
self._cronDTC[channel].expanded, self._cronDTC[channel].nth_weekday_of_month = croniter.expand(' '.join(items[:5]))
else:
serv.privmsg(channel, 'cronDTC : update refuse !')
self.__prompt('host', '{%alert%}cronDTC : update refuse !{%default%}')
elif message == 'next':
if self._cronDTC[channel].is_alive():
_next = delai(datetime.fromtimestamp(self._cronDTC[channel].get_current()))
serv.privmsg(channel, 'cronDTC : next in {next} !'.format(next = _next))
self.__prompt('host', '{%alert%}cronDTC : next in {next} !{%default%}', next = _next)
else:
return False
else:
if not message:
self._cronDTC[channel] = cron(target = self.do_dtc, expr_format = '0 * * * *', args = (self._name, channel, serv, ''))
elif croniter.is_valid(' '.join(items[:5])):
self._cronDTC[channel] = cron(target = self.do_dtc, expr_format = ' '.join(items[:5]), args = (self._name, channel, serv, ''))
else:
return False
self._cronDTC[channel].start()
serv.privmsg(channel, 'cronDTC : start !')
self.__prompt('host', '{%alert%}cronDTC : start !{%default%}')
return True
def validate(self, attrs):
if 'schedule_type' in attrs:
schedule_type = attrs['schedule_type']
else:
schedule_type = self.instance['schedule_type']
value = attrs.get('schedule')
if not value:
return attrs
if schedule_type == ScheduleType.INTERVAL:
# type: [int count, str unit name]
if not isinstance(value, list):
raise ValidationError('Invalid value for schedule_type')
if not isinstance(value[0], int):
raise ValidationError(
'Invalid value for schedule unit count (index 0)')
if value[1] not in INTERVAL_NAMES:
raise ValidationError(
'Invalid value for schedule unit name (index 1)')
elif schedule_type == ScheduleType.CRONTAB:
# type: str schedule
if not isinstance(value, six.string_types):
raise ValidationError('Invalid value for schedule_type')
value = value.strip()
if value.startswith('@'):
try:
value = NONSTANDARD_CRONTAB_SCHEDULES[value]
except KeyError:
raise ValidationError('Schedule was not parseable')
if not croniter.is_valid(value):
raise ValidationError('Schedule was not parseable')
attrs['schedule'] = value
return attrs
def __init__(self,
id,
update,
interval,
name,
user_id,
tz,
command,
context=[],
last_run=None,
next_run=None,
total_run_count=0,
*args,
**kwargs):
self._id = id
self._update = update
self._name = name
self._user_id = user_id
self._tz = tz
# validate cron interval
if not croniter.is_valid(interval):
raise ValueError("The 'interval' value is not valid")
self._interval = interval
self._command = command
self._context = context
self._last_run = last_run
self._next_run = next_run
self._total_run_count = total_run_count
def crontab_config_filter(self, trigger_time, crontab_config):
'''
Crontab执行时机过滤函数
算法描述如下：
trigger_time - 1m trigger_time
| | 实际Starter执行时间（比trigger_time晚1秒，此时计算出trigger_time）
| | 1s |
| | |
+-----------------------------+====+------------> Future
^ ^
| |
| start_time（需要执行的Crontab任务）
|
start_time（不需要执行的Crontab任务）
即：只有启动点大于等于当前触发点的才需要执行
'''
crontab_expr = crontab_config['crontab']
if not crontab_expr:
return False
if not croniter.is_valid(crontab_expr):
return False
now = arrow.get(trigger_time + 1).to('Asia/Shanghai').datetime
crontab_iter = croniter(crontab_expr, now)
start_time = int(crontab_iter.get_prev())
return start_time >= trigger_time
def create_cron_job(cron_job):
# check cron_job does't exists first
cron_job_exists = mongo_connection.mongo_check_cron_job_exists(cron_job)
if cron_job_exists is True:
return jsonify({"cron_job_exists": True}), 403
else:
# check the request is passed with all needed parameters
try:
cron_job_json = request.json
schedule = cron_job_json["schedule"]
env_vars = return_sane_default_if_not_declared(
"env_vars", cron_job_json, {})
docker_image = cron_job_json["docker_image"]
running = return_sane_default_if_not_declared(
"running", cron_job_json, True)
networks = return_sane_default_if_not_declared(
"networks", cron_job_json, [])
volumes = return_sane_default_if_not_declared(
"volumes", cron_job_json, [])
devices = return_sane_default_if_not_declared(
"devices", cron_job_json, [])
privileged = return_sane_default_if_not_declared(
"privileged", cron_job_json, False)
except:
return json.dumps(
find_missing_params(cron_job_json,
["docker_image", "schedule"])), 400
# check edge case where schedule is not valid
if croniter.is_valid(schedule) is False:
return jsonify({"schedule_valid": False}), 400
# update the db
cron_job_json = mongo_connection.mongo_add_cron_job(
cron_job, schedule, env_vars, docker_image, running, networks,
volumes, devices, privileged)
return dumps(cron_job_json), 200
def post(self, project_id):
'''
Create new cronjob
'''
b = request.get_json()
if not CronJobs.name_pattern.match(b['name']):
abort(400, 'CronJob name must be not empty alphanumeric string.')
if not croniter.is_valid('%s %s %s %s %s' % (b['minute'], b['hour'], b['day_month'], b['month'], b['day_week'])):
abort(400, 'Invalid input expression')
result = g.db.execute_one_dict("""
SELECT COUNT(*) as cnt FROM cronjob WHERE project_id = %s
""", [project_id])
if result['cnt'] > 50:
abort(400, 'Too many cronjobs.')
r = g.db.execute_one("""
SELECT count(*) FROM cronjob
WHERE project_id = %s AND name = %s
""", [project_id, b['name']])
if r[0] > 0:
abort(400, 'CronJob with this name already exist')
g.db.execute('''
INSERT INTO cronjob (project_id, name, minute, hour, day_month, month, day_week, sha, infrabox_file) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s)
''', [project_id, b['name'], b['minute'], b['hour'], b['day_month'], b['month'], b['day_week'], b['sha'], b['infrabox_file']])
g.db.commit()
return OK('Successfully added CronJob.')
def validate(self, attrs):
if "schedule_type" in attrs:
schedule_type = attrs["schedule_type"]
else:
schedule_type = self.instance["schedule_type"]
value = attrs.get("schedule")
if not value:
return attrs
if schedule_type == ScheduleType.INTERVAL:
if not isinstance(value, list):
raise ValidationError("Invalid value for schedule_type")
if not isinstance(value[0], int):
raise ValidationError(
"Invalid value for schedule unit count (index 0)")
if value[1] not in INTERVAL_NAMES:
raise ValidationError(
"Invalid value for schedule unit name (index 1)")
elif schedule_type == ScheduleType.CRONTAB:
if not isinstance(value, six.string_types):
raise ValidationError("Invalid value for schedule_type")
value = value.strip()
if value.startswith("@"):
try:
value = NONSTANDARD_CRONTAB_SCHEDULES[value]
except KeyError:
raise ValidationError("Schedule was not parseable")
if not croniter.is_valid(value):
raise ValidationError("Schedule was not parseable")
attrs["schedule"] = value
return attrs
def validate_cronjob_format(schedule):
"""
validates if the cronjob schedule provided is valid
:param schedule: input schedule
:return: True/False
"""
return croniter.is_valid(schedule)
def validate_schedule(self, attrs, source):
if 'schedule_type' in attrs:
schedule_type = attrs['schedule_type']
else:
schedule_type = self.object['schedule_type']
value = attrs[source]
if not value:
return attrs
if schedule_type == ScheduleType.INTERVAL:
if not isinstance(value, list):
raise ValidationError('Invalid value for schedule_type')
if not isinstance(value[0], int):
raise ValidationError('Invalid value for schedule frequency')
if value[1] not in INTERVAL_NAMES:
raise ValidationError('Invalid value for schedlue interval')
elif schedule_type == ScheduleType.CRONTAB:
if not isinstance(value, six.string_types):
raise ValidationError('Invalid value for schedule_type')
value = value.strip()
if value.startswith('@'):
try:
value = NONSTANDARD_CRONTAB_SCHEDULES[value]
except KeyError:
raise ValidationError('Schedule was not parseable')
if not croniter.is_valid(value):
raise ValidationError('Schedule was not parseable')
attrs[source] = value
return attrs
def update_cron_job_all_fields(cron_job):
# check cron_job exists first
cron_job_exists = mongo_connection.mongo_check_cron_job_exists(cron_job)
if cron_job_exists is False:
return jsonify({"cron_job_exists": False}), 403
# check cron_job got update parameters
try:
cron_job_json = request.json
if len(cron_job_json) == 0:
return jsonify({"missing_parameters": True}), 400
if cron_job_json["docker_image"] is None or cron_job_json["schedule"] is None:
return json.dumps(find_missing_params(cron_job_json, ["docker_image", "schedule"])), 400
except:
return jsonify({"missing_parameters": True}), 400
# set default for undeclared params
try:
cron_job_json["env_vars"] = return_sane_default_if_not_declared("env_vars", cron_job_json, {})
cron_job_json["running"] = return_sane_default_if_not_declared("running", cron_job_json, True)
cron_job_json["volumes"] = return_sane_default_if_not_declared("volumes", cron_job_json, [])
cron_job_json["devices"] = return_sane_default_if_not_declared("devices", cron_job_json, [])
cron_job_json["privileged"] = return_sane_default_if_not_declared("privileged", cron_job_json, False)
cron_job_json["networks"] = return_sane_default_if_not_declared("networks", cron_job_json, ["nebula", "bridge"])
except:
return json.dumps(find_missing_params(cron_job_json, ["docker_image", "schedule"])), 400
# check edge case of port being outside of possible port ranges in case trying to update port listing
try:
schedule = request.json["schedule"]
# check edge case where schedule is not valid
if croniter.is_valid(schedule) is False:
return jsonify({"schedule_valid": False}), 400
except:
pass
# update db
cron_job_json = mongo_connection.mongo_update_cron_job_fields(cron_job, request.json)
return dumps(cron_job_json), 202
def __init__(self, config):
self._startup_run_done = False
if config.schedule is None:
self.mode = ScheduleMode.once
self.run_at_startup = True
else:
try:
self.interval = int(config.schedule)
self.cron = None
self.mode = ScheduleMode.interval
self.run_at_startup = config.run_at_startup if config.run_at_startup is not None else True
except ValueError:
self.interval = None
self.cron = config.schedule
self.mode = ScheduleMode.cron
self.run_at_startup = config.run_at_startup if config.run_at_startup is not None else False
if self.mode == ScheduleMode.interval:
if self.interval <= 0:
raise ValueError("Interval must be at least one second!")
if self.mode == ScheduleMode.cron:
self.schedule_hash_id = str.encode(
config.schedule_hash_id if config.schedule_hash_id is not None else socket.getfqdn())
if not croniter.is_valid(self.cron, hash_id=self.schedule_hash_id):
raise ValueError("Invalid cron expression!")
def __init__(self, task=None, schedule=None, args=(), kwargs={}):
# Task needs to be callable
if not callable(task):
raise TypeError("{} is not callable".format(task))
# Validate cron schedule
if not croniter.is_valid(schedule):
raise ValueError(
"'{}' is not a valid cron string.".format(schedule))
# Validate args
if args is not None and not isinstance(args, tuple) and not isinstance(
args, list):
raise InvalidSignatureException(
"Args must be a tuple or list, got {}".format(args))
# Marshall args to tuple
if isinstance(args, list):
args = tuple(args)
# Validate kwargs
if kwargs is not None and not isinstance(kwargs, dict):
raise InvalidSignatureException(
"Kwargs must be a dict, got {}".format(kwargs))
self.task = task
self.schedule = schedule
self.next_run = datetime.fromtimestamp(
croniter(schedule, datetime.now()).get_next())
self.args = args
self.kwargs = kwargs
async def check_schedule(self, ctx):
table = []
for k, v in self.userdata.items():
if ctx.guild.id != v["server_id"]:
continue
# get next running
run_at = self._strftime(
croniter(v["schedule"],
self._now()).get_next(dt)) if croniter.is_valid(
v["schedule"]) else None
# get target channel
run_on = self.bot.get_channel(
v["channel_id"]
) if "channel_id" in v.keys() and v["channel_id"] else None
# get author
author = await ctx.guild.fetch_member(v["author"])
author_name = author.nick if author else "Unknown"
table.append([run_at, run_on, k, author_name])
if len(table):
formatted_table = tabulate(
table, ["次回実行日時", "実行先チャンネル", "スケジュール名", "登録した人"],
tablefmt="simple")
await ctx.send(
f":bulb: 直近の実行タイミングは次の通りです。\n```{formatted_table}```")
else:
await ctx.send(":bulb: 登録されているスケジュールはありません。")
def __call__(self, value) -> typing.Any:
message = f'The schedule_interval expression `{value}` must be a valid CRON expression: ' \
'validate it here https://crontab.guru/'
if not croniter.is_valid(value):
raise validate.ValidationError(message)
return value
def __init__(self, cron: str = None):
# validates if cron is a valid string
if cron and not croniter.is_valid(cron):
raise ValueError(f'''
Invalid cron string: `{cron}`. For help, consult `crontab.guru.com`
''')
self.cron = cron
def validate_schedule(value):
'''
校验 schedule 是否合法
'''
if croniter.is_valid(value):
return value
raise ValidationError
def validate_schedule(self, attrs, source):
if 'schedule_type' in attrs:
schedule_type = attrs['schedule_type']
else:
schedule_type = self.object['schedule_type']
value = attrs[source]
if not value:
return attrs
if schedule_type == ScheduleType.INTERVAL:
# type: [int count, str unit name]
if not isinstance(value, list):
raise ValidationError('Invalid value for schedule_type')
if not isinstance(value[0], int):
raise ValidationError('Invalid value for schedule unit count (index 0)')
if value[1] not in INTERVAL_NAMES:
raise ValidationError('Invalid value for schedlue unit name (index 1)')
elif schedule_type == ScheduleType.CRONTAB:
# type: str schedule
if not isinstance(value, six.string_types):
raise ValidationError('Invalid value for schedule_type')
value = value.strip()
if value.startswith('@'):
try:
value = NONSTANDARD_CRONTAB_SCHEDULES[value]
except KeyError:
raise ValidationError('Schedule was not parseable')
if not croniter.is_valid(value):
raise ValidationError('Schedule was not parseable')
attrs[source] = value
return attrs
def _check_schedule(self):
if self.interval and self.schedule:
raise InvalidQuerySchedule(
self.name, "both interval and schedule specified"
)
if self.schedule and not croniter.is_valid(self.schedule):
raise InvalidQuerySchedule(self.name, "invalid schedule format")
def wrapper(*args, **kwargs):
cron = kwargs.get('cron', None)
if not isinstance(cron, str):
raise TypeError("invalid cron")
if not croniter.is_valid(cron):
raise ValueError("Invalid cron specified {}".format(cron))
return func(*args, **kwargs)
async def add_schedule(self, ctx, name, m, h, dom, mon, dow, *cmd):
crontab = " ".join([m, h, dom, mon, dow])
escaped_cmds = None
if not croniter.is_valid(crontab):
await ctx.send(f":x: スケジュール `{crontab}` が不正です。")
return
else:
if not self._dig(list(cmd)):
await ctx.send(
":x: 他Botのコマンドまたは正しくないコマンドは定時実行ができないので登録しませんでした。")
logger.debug("Specified command could not scheduled.")
return
escaped_cmds = " ".join((f'"{i}"' if " " in i else i for i in cmd))
self.userdata[name] = {
"command": escaped_cmds,
"server_id": ctx.guild.id,
"channel_id": ctx.channel.id,
"author": ctx.message.author.id,
"schedule": crontab,
}
await self._save_userdata()
logger.debug("Specified command scheduled successfully.")
next_run = self._strftime(croniter(crontab, self._now()).get_next(dt))
await ctx.send(f":white_check_mark: スケジュール `{name}` を追加しました。\n"
f"次回実行予定は {next_run} です。\n"
f"実行されるコマンドは `{escaped_cmds}` です。")
def check_invalid(window, values):
invalid = False
if re.search(
"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+",
window["link"].get(),
):
window["invalid_link"].update(visible=False)
else:
invalid = True
window["invalid_link"].update(visible=True)
if re.search("[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+",
window["recipient"].get()):
window["invalid_recipient"].update(visible=False)
else:
window["invalid_recipient"].update(visible=True)
invalid = True
if values["content_type"] in ["text/plain", "text/html"]:
window["invalid_type"].update(visible=False)
else:
window["invalid_type"].update(visible=True)
invalid = True
if croniter.is_valid(values["frequency"]):
window["invalid_cron"].update(visible=False)
else:
window["invalid_cron"].update(visible=True)
invalid = True
return invalid
def _update_current_crontab_schedule(self):
# select not execute record from journal
query = sql.SQL('SELECT {} FROM {} WHERE {}').format(
sql.SQL(', ').join(map(sql.Identifier, ('id', 'type', 'plan'))),
sql.Identifier(self._table_name),
sql.SQL('{} = {} AND {} > {} AND {} is Null').format(
sql.Identifier('status'), sql.Literal(self.JobStatus.TODO),
sql.Identifier('plan'), sql.Literal(datetime.now()),
sql.Identifier('params')
)
)
cursor = self._db_conn.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor)
cursor.execute(query)
rows = cursor.fetchall()
rows = rows if rows else list() # change None to empty list
# id, type, start and valid state (used later)
schedule = [[row.id, row.type, row.plan, False] for row in rows]
# check current crontab schedule from script
for activity_type, params in self._registry.items():
cron_tab = params['crontab'] or ''
if not croniter.is_valid(cron_tab):
continue
next_plan = croniter(params['crontab'], datetime.now()).get_next(datetime)
need_to_plan = True
for rec in schedule:
if rec[1] == activity_type:
if next_plan == rec[2]:
rec[3] = True
need_to_plan = False
break
if need_to_plan:
insert_params = {
'type': activity_type,
'status': 'todo',
'plan': next_plan,
}
insert_new_schedule_query =\
sql.SQL('insert into {}({}) values({})').format(
sql.Identifier(self._table_name),
sql.SQL(', ').join(sql.Identifier(field) for field in insert_params.keys()),
sql.SQL(', ').join(sql.Literal(value) for value in insert_params.values())
)
cursor.execute(insert_new_schedule_query)
self._db_conn.commit()
# prepare journal record list for removing
rec_for_delete = [rec[0] for rec in schedule if not rec[3]]
# and removing them
if len(rec_for_delete):
delete_not_valid_record_query = sql.SQL('DELETE FROM {} WHERE {} IN ({})').format(
sql.Identifier(self._table_name),
sql.Identifier('id'),
sql.SQL(', ').join(sql.Literal(_id) for _id in rec_for_delete)
)
cursor.execute(delete_not_valid_record_query)
self._db_conn.commit()
def job_load(entry, job):
jid = None
if 'run_interval' in job or 'run_cron' in job:
if 'id' in job:
jid = job['id']
del job['id']
if not jid or jid in entry.data['jobs']:
#jid = ((job['group'] + '.') if 'group' in job and job['group'] else ((job['entry_id'] + '.') if 'entry_id' in job and job['entry_id'] else '')) + hashlib.sha1((str(i) + ':' + str(job)).encode('UTF-8')).hexdigest()[:16]
i = 0
while True:
jid = ((job['group'] +
'.') if 'group' in job and job['group'] else
((job['entry_id'] + '.') if 'entry_id' in job
and job['entry_id'] else '')) + hashlib.sha1(
(str(job)).encode('UTF-8')).hexdigest()[:16] + (
('_' + str(i)) if i else '')
if not (jid in entry.data['jobs']):
break
i = i + 1
if jid in entry.scheduler_oldjobs:
job = {**entry.scheduler_oldjobs[jid], **job}
if 'do' in job and isinstance(job['do'], str):
job['do'] = [job['do']]
if 'enabled' not in job:
job['enabled'] = True
if 'max_delay' not in job:
job['max_delay'] = 60 if 'run_cron' in job else 0
job['max_delay'] = utils.read_duration(job['max_delay'])
if 'timer_to' not in job:
job['timer_to'] = 0
if 'last_run' not in job:
job['last_run'] = 0
if 'run_interval' in job:
job['run_interval'] = utils.read_duration(job['run_interval'])
if job['run_interval'] <= 0:
job = False
if job and 'run_cron' in job and not croniter.is_valid(
job['run_cron']):
logging.error(
'#{id}> invalid cron rule: {cron} in job: {job}'.format(
id=entry.id, cron=job['run_cron'], job=job))
job = False
if job:
if 'next_run' not in job or (
job['max_delay'] > 0
and system.time() >= job['next_run'] + job['max_delay']):
job_set_next_run(job)
entry.data['jobs'][jid] = job
if 'group' in job and job[
'group'] and not job['group'] in entry.data['groups']:
entry.data['groups'][job['group']] = {
'enabled': True,
'timer_to': 0
} if job[
'group'] not in entry.scheduler_oldgroups else entry.scheduler_oldgroups[
job['group']]
return jid
def __init__(
self, cron: str, start_date: datetime = None, end_date: datetime = None
):
# build cron object to check the cron string - will raise an error if it's invalid
if not croniter.is_valid(cron):
raise ValueError("Invalid cron string: {}".format(cron))
self.cron = cron
super().__init__(start_date=start_date, end_date=end_date)
def interval(self, value):
if value is None:
raise ValueError("The 'interval' can not be None")
if not croniter.is_valid(value):
raise ValueError("The 'interval' value is not valid")
self._interval = value
def pre_add(self, item: "AlertModelView") -> None:
item.recipients = get_email_address_str(item.recipients)
if not croniter.is_valid(item.crontab):
raise SupersetException("Invalid crontab format")
item.validator_type = item.validator_type.lower()
check_validator(item.validator_type, item.validator_config)
def __init__(self, name, cron, kids=False):
if not croniter.is_valid(cron):
raise ValueError(
"cron was not valid cron sytax. Should be in min hour day" +
" month day_of_week, instead was {}".format(cron))
self.name = name
self.cron = cron
self.kids = kids
def is_cron(key, val):
"""
Check crontab syntax or ``None``.
"""
if val is None:
return
assert croniter.is_valid(val), "[{}] expected cron syntax or None".format(
key)
def main():
parser = argparse.ArgumentParser(
description='Tool for running and initializing backup container')
parser.add_argument('-i',
'--initialize',
action='store_true',
help='configure cron and install dependencies')
parser.add_argument('-r',
'--run',
action='store_true',
help='runs scrips following the lifecycle')
parser.add_argument('-csp',
'--container-scripts-path',
default='/scripts',
help='path to scripts folder, default /scripts')
parser.add_argument(
'-c',
'--cron-path',
default='/etc/crontabs/root',
help='path to scripts folder, default /etc/crontabs/root')
arguments = parser.parse_args()
if CRON_BACKUP_SCHEDULE not in os.environ:
print_logger("Could not find '%s' in the environment variables" %
CRON_BACKUP_SCHEDULE)
exit(1)
if not croniter.is_valid(os.environ[CRON_BACKUP_SCHEDULE]):
print_logger("Provided cron schedule '%s' is not valid" %
os.environ[CRON_BACKUP_SCHEDULE])
exit(1)
# check if provided directory exists
if not os.path.isdir(arguments.container_scripts_path):
print_logger("The following path '%s' is not a valid directory" %
arguments.container_scripts_path)
exit(1)
if not arguments.initialize and not arguments.run:
print_logger(
"Nothing to do.\nPlease run: 'fridge --help' to see possible options"
)
exit(1)
scripts_detector = ScriptsDetector(arguments.container_scripts_path)
scripts_detector.list_detected_scripts()
if arguments.initialize:
print_logger("Initializing...")
start_initialize(scripts_detector, arguments.cron_path)
print_logger("Initialization complete.")
if arguments.run:
print_logger("Running...")
start_run(scripts_detector)
print_logger("Finished running.")
def pre_add(self, obj):
try:
recipients = get_email_address_list(obj.recipients)
obj.recipients = ', '.join(recipients)
except Exception:
raise SupersetException('Invalid email list')
obj.user = obj.user or g.user
if not croniter.is_valid(obj.crontab):
raise SupersetException('Invalid crontab format')
def pre_add(self, obj):
try:
recipients = get_email_address_list(obj.recipients)
obj.recipients = ", ".join(recipients)
except Exception:
raise SupersetException("Invalid email list")
obj.user = obj.user or g.user
if not croniter.is_valid(obj.crontab):
raise SupersetException("Invalid crontab format")
def __init__(self, target, args, expr_format):
if croniter.is_valid(expr_format):
Thread.__init__(self)
self.__target = target
self.__args = args
croniter.__init__(self, expr_format)
self.get_next()
self.__forceStop = False
self.__initialized = True
else:
self = None
def _validate(self, name, start, schedule_type, arguments, repeat, duration, end):
if name is None or not isinstance(name, basestring) or name.strip() == '':
raise RuntimeError('A schedule must have a name')
# Check whether the requested type is valid
accepted_types = ['GROUP_ACTION', 'BASIC_ACTION', 'LOCAL_API']
if schedule_type not in accepted_types:
raise RuntimeError('Unknown schedule type. Allowed: {0}'.format(', '.join(accepted_types)))
# Check duration/repeat/end combinations
if repeat is None:
if end is not None:
raise RuntimeError('No `end` is allowed when it is a non-repeated schedule')
else:
if not croniter.is_valid(repeat):
raise RuntimeError('Invalid `repeat`. Should be a cron-style string. See croniter documentation')
if duration is not None and duration <= 60:
raise RuntimeError('If a duration is specified, it should be at least more than 60s')
# Type specifc checks
if schedule_type == 'BASIC_ACTION':
if duration is not None:
raise RuntimeError('A schedule of type BASIC_ACTION does not have a duration. It is a one-time trigger')
if not isinstance(arguments, dict) or 'action_type' not in arguments or not isinstance(arguments['action_type'], int) or \
'action_number' not in arguments or not isinstance(arguments['action_number'], int) or len(arguments) != 2:
raise RuntimeError('The arguments of a BASIC_ACTION schedule must be of type dict with arguments `action_type` and `action_number`')
elif schedule_type == 'GROUP_ACTION':
if duration is not None:
raise RuntimeError('A schedule of type GROUP_ACTION does not have a duration. It is a one-time trigger')
if not isinstance(arguments, int) or arguments < 0 or arguments > 254:
raise RuntimeError('The arguments of a GROUP_ACTION schedule must be an integer, representing the Group Action to be executed')
elif schedule_type == 'LOCAL_API':
if duration is not None:
raise RuntimeError('A schedule of type LOCAL_API does not have a duration. It is a one-time trigger')
if not isinstance(arguments, dict) or 'name' not in arguments or 'parameters' not in arguments or not isinstance(arguments['parameters'], dict):
raise RuntimeError('The arguments of a LOCAL_API schedule must be of type dict with arguments `name` and `parameters`')
func = getattr(self._web_interface, arguments['name']) if hasattr(self._web_interface, arguments['name']) else None
if func is None or not callable(func) or not hasattr(func, 'plugin_exposed') or getattr(func, 'plugin_exposed') is False:
raise RuntimeError('The arguments of a LOCAL_API schedule must specify a valid and (plugin_)exposed call')
check = getattr(func, 'check')
if check is not None:
params_parser(arguments['parameters'], check)
def trigger(site, event, last=None, queued_jobs=(), now=False):
"""Trigger method in hooks.scheduler_events."""
queue = 'long' if event.endswith('_long') else 'short'
timeout = queue_timeout[queue]
if not queued_jobs and not now:
queued_jobs = get_jobs(site=site, queue=queue)
if frappe.flags.in_test:
frappe.flags.ran_schedulers.append(event)
events_from_hooks = get_scheduler_events(event)
if not events_from_hooks:
return
events = events_from_hooks
if not now:
events = []
if event == "cron":
for e in events_from_hooks:
e = cron_map.get(e, e)
if croniter.is_valid(e):
if croniter(e, last).get_next(datetime) <= frappe.utils.now_datetime():
events.extend(events_from_hooks[e])
else:
frappe.log_error("Cron string " + e + " is not valid", "Error triggering cron job")
frappe.logger(__name__).error('Exception in Trigger Events for Site {0}, Cron String {1}'.format(site, e))
else:
if croniter(cron_map[event], last).get_next(datetime) <= frappe.utils.now_datetime():
events.extend(events_from_hooks)
for handler in events:
if not now:
if handler not in queued_jobs:
enqueue(handler, queue, timeout, event)
else:
scheduler_task(site=site, event=event, handler=handler, now=True)
def test_invalid_zerorepeat(self):
self.assertFalse(croniter.is_valid('*/0 * * * *'))
def test_is_valid(self):
self.assertTrue(croniter.is_valid('0 * * * *'))
self.assertFalse(croniter.is_valid('0 * *'))
self.assertFalse(croniter.is_valid('* * * janu-jun *'))