def test_croniter_range_derived_class(self):
# trivial example extending croniter
class croniter_nosec(croniter):
""" Like croniter, but it forbids second-level cron expressions. """
@classmethod
def expand(cls, expr_format, *args, **kwargs):
if len(expr_format.split()) == 6:
raise CroniterBadCronError(
"Expected 'min hour day mon dow'")
return croniter.expand(expr_format, *args, **kwargs)
cron = "0 13 8 1,4,7,10 wed"
matches = list(
croniter_range(datetime(2020, 1, 1),
datetime(2020, 12, 31),
cron,
day_or=False,
_croniter=croniter_nosec))
self.assertEqual(len(matches), 3)
cron = "0 1 8 1,15,L wed 15,45"
with self.assertRaises(CroniterBadCronError):
# Should fail using the custom class that forbids the seconds expression
croniter_nosec(cron)
with self.assertRaises(CroniterBadCronError):
# Should similarly fail because the custom class rejects seconds expr
i = croniter_range(datetime(2020, 1, 1),
datetime(2020, 12, 31),
cron,
_croniter=croniter_nosec)
next(i)
def test_input_type_exceptions(self):
dt_start1 = datetime(2019, 1, 1)
dt_stop1 = datetime(2020, 1, 1)
f_start1 = 1552252218.0
f_stop1 = 1591823311.0
# Mix start/stop types
with self.assertRaises(TypeError):
list(croniter_range(dt_start1, f_stop1, "0 * * * *"), ret_type=datetime)
with self.assertRaises(TypeError):
list(croniter_range(f_start1, dt_stop1, "0 * * * *"))
def test_issue145_range(self):
cron = "0 13 8 1,4,7,10 wed"
matches = list(croniter_range(datetime(2020, 1, 1), datetime(2020, 12, 31), cron, day_or=False))
self.assertEqual(len(matches), 3)
self.assertEqual(matches[0], datetime(2020, 1, 8, 13))
self.assertEqual(matches[1], datetime(2020, 4, 8, 13))
self.assertEqual(matches[2], datetime(2020, 7, 8, 13))
# No matches within this range; therefore expect empty list
matches = list(croniter_range(datetime(2020, 9, 30), datetime(2020, 10, 30), cron, day_or=False))
self.assertEqual(len(matches), 0)
def test_1day_step_no_ends(self):
# Test without ends (exclusive)
start = datetime(2016, 12, 2)
stop = datetime(2016, 12, 10)
fwd = list(croniter_range(start, stop, '0 0 * * *', exclude_ends=True))
self.assertEqual(len(fwd), 7)
self.assertNotEqual(fwd[0], start)
self.assertNotEqual(fwd[-1], stop)
# Test the same, but in reverse
rev = list(croniter_range(stop, start, '0 0 * * *', exclude_ends=True))
self.assertEqual(len(rev), 7)
self.assertNotEqual(fwd[0], stop)
self.assertNotEqual(fwd[-1], start)
def test_1day_step(self):
start = datetime(2016, 12, 2)
stop = datetime(2016, 12, 10)
fwd = list(croniter_range(start, stop, '0 0 * * *'))
self.assertEqual(len(fwd), 9)
self.assertEqual(fwd[0], start)
self.assertEqual(fwd[-1], stop)
# Test the same, but in reverse
rev = list(croniter_range(stop, start, '0 0 * * *'))
self.assertEqual(len(rev), 9)
# Ensure forward/reverse are a mirror image
rev.reverse()
self.assertEqual(fwd, rev)
def test_1minute_step_float(self):
start = datetime(2000, 1, 1, 0, 0)
stop = datetime(2000, 1, 1, 0, 1)
res = list(croniter_range(start, stop, '* * * * *', ret_type=float))
self.assertEqual(len(res), 2)
self.assertEqual(res[0], 946684800.0)
self.assertEqual(res[-1] - res[0], 60)
def test_dt_types(self):
start = mydatetime(2020, 9, 24)
stop = datetime(2020, 9, 28)
try:
list(croniter_range(start, stop, '0 0 * * *'))
except CroniterBadTypeRangeError:
self.fail('should not be triggered')
def launch_scheduled_runs_for_schedule(instance,
schedule_state,
end_datetime_utc,
max_catchup_runs,
debug_crash_flags=None):
check.inst_param(instance, "instance", DagsterInstance)
check.inst_param(schedule_state, "schedule_state", ScheduleState)
check.inst_param(end_datetime_utc, "end_datetime_utc", datetime.datetime)
latest_tick = instance.get_latest_tick(schedule_state.schedule_origin_id)
if not latest_tick:
start_timestamp_utc = schedule_state.start_timestamp
elif latest_tick.status == ScheduleTickStatus.STARTED:
# Scheduler was interrupted while performing this tick, re-do it
start_timestamp_utc = latest_tick.timestamp
else:
start_timestamp_utc = latest_tick.timestamp + 1
start_datetime_utc = datetime.datetime.fromtimestamp(start_timestamp_utc,
tz=get_utc_timezone())
tick_times = list(
croniter_range(start_datetime_utc, end_datetime_utc,
schedule_state.cron_schedule))
for schedule_time_utc in tick_times[-max_catchup_runs:]:
if latest_tick and latest_tick.timestamp == schedule_time_utc.timestamp(
):
tick = latest_tick
else:
tick = instance.create_schedule_tick(
ScheduleTickData(
schedule_origin_id=schedule_state.schedule_origin_id,
schedule_name=schedule_state.name,
timestamp=schedule_time_utc.timestamp(),
cron_schedule=schedule_state.cron_schedule,
status=ScheduleTickStatus.STARTED,
))
_check_for_debug_crash(debug_crash_flags, "TICK_CREATED")
with ScheduleTickHolder(tick, instance) as tick_holder:
_check_for_debug_crash(debug_crash_flags, "TICK_HELD")
with RepositoryLocationHandle.create_from_repository_origin(
schedule_state.origin.repository_origin,
instance) as repo_location_handle:
repo_location = RepositoryLocation.from_handle(
repo_location_handle)
_schedule_run_at_time(
instance,
repo_location,
schedule_state,
schedule_time_utc,
tick_holder,
debug_crash_flags,
)
def test_timezone_dst(self):
""" Test across DST transition, which technially is a timzone change. """
tz = pytz.timezone("US/Eastern")
start = tz.localize(datetime(2020, 10, 30))
stop = tz.localize(datetime(2020, 11, 10))
res = list(croniter_range(start, stop, '0 0 * * *'))
self.assertNotEqual(res[0].tzinfo, res[-1].tzinfo)
self.assertEqual(len(res), 12)
def test_auto_ret_type(self):
data = [
(datetime(2019, 1, 1), datetime(2020, 1, 1), datetime),
(1552252218.0, 1591823311.0, float),
]
for start, stop, rtype in data:
ret = list(croniter_range(start, stop, "0 0 * * *"))
self.assertIsInstance(ret[0], rtype)
def test_1month_step(self):
start = datetime(1982, 1, 1)
stop = datetime(1983, 12, 31)
res = list(croniter_range(start, stop, '0 0 1 * *'))
self.assertEqual(len(res), 24)
self.assertEqual(res[0], start)
self.assertEqual(res[5].day, 1)
self.assertEqual(res[-1], datetime(1983, 12, 1))
def analyze_video(self, file, start, end, increment, update_expression):
result = {}
# run ffmpeg.probe to get the frame rate and frame count
videoInfo = utils.get_video_info(file)
startFrame = utils.seconds_to_frames(start, videoInfo['fps'])
videoInfo['frame_count'] = videoInfo[
'frame_count'] - utils.seconds_to_frames(end, videoInfo['fps'])
# print some initial information
logging.info(f"Analyzing {file}")
result['fps'] = videoInfo['fps']
result['runtime'] = videoInfo['runtime'] / 60
# video name, no ext
video_name = os.path.splitext(os.path.basename(file))[0]
result['file'] = video_name
# check if we have a "save" file
currentPosition = startFrame
saveFile = utils.read_db(self.db, utils.DB_LAST_PLAYED_FILE)
if (file == saveFile['file']):
currentPosition = float(saveFile['pos'])
if (currentPosition < startFrame):
currentPosition = startFrame
# find total time to play entire movie
result['total_time_to_play'] = self.time_to_play(
videoInfo['frame_count'] - startFrame, float(increment),
update_expression)
# find time to play what's left
result['remaining_time_to_play'] = self.time_to_play(
videoInfo['frame_count'] - currentPosition, float(increment),
update_expression)
# figure out how many 'real time' minutes per hour
now = datetime.now()
tomorrow = now + timedelta(days=1)
day_total = 0
for i in croniter_range(now, tomorrow, update_expression):
day_total = day_total + 1
secondsPerIncrement = utils.frames_to_seconds(increment,
videoInfo['fps'])
# this is how many "seconds" of film actually shown per second of realtime
framesPerSecond = secondsPerIncrement / (60 / (day_total / 24) * 60)
minutesPerHour = (framesPerSecond * 60)
result['minutes_per_hour'] = minutesPerHour
result['minutes_per_day'] = minutesPerHour * 24
# number of frames left to play
result['frames_left'] = videoInfo['frame_count'] - currentPosition
return result
def test_extra_hour_day_prio(self):
def datetime_tz(*args, **kw):
""" Defined this in another branch. single-use-version """
tzinfo = kw.pop("tzinfo")
return tzinfo.localize(datetime(*args))
tz = pytz.timezone("US/Eastern")
cron = "0 3 * * *"
start = datetime_tz(2020, 3, 7, tzinfo=tz)
end = datetime_tz(2020, 3, 11, tzinfo=tz)
ret = [i.isoformat() for i in croniter_range(start, end, cron)]
self.assertEqual(ret, [
"2020-03-07T03:00:00-05:00", "2020-03-08T03:00:00-04:00",
"2020-03-09T03:00:00-04:00", "2020-03-10T03:00:00-04:00"
])
def execute(self):
"""Execute the schedule.
A check is done to make sure the schedule wasn't checked by another worker in the mean time.
The task may be added several times if catch_up is True.
Returns True if at least one task was queued (so you can loop for testing).
"""
last_check = self.last_check
self.refresh_from_db()
if last_check != self.last_check:
# this schedule was executed from another worker in the mean time
return True
# we update last_check already to reduce race condition chance
self.last_check = timezone.now()
self.save()
did_something = False
if last_check is None:
next_dues = [croniter(self.cron, timezone.now()).get_prev(datetime)]
else:
next_dues = list(croniter_range(last_check, timezone.now(), self.cron))
if not self.catch_up and len(next_dues) > 1:
next_dues = [next_dues[-1]]
for next_due in next_dues:
logger.debug(f"Due : {self}")
dt_kwarg = {}
if self.datetime_kwarg:
dt_kwarg = {self.datetime_kwarg: next_due}
t = tasks[self.function].queue(*self.args, **self.kwargs, **dt_kwarg)
if t:
self.last_run = t
self.save()
did_something = True
return did_something
def launch_scheduled_runs_for_schedule(instance,
logger,
schedule_state,
end_datetime_utc,
max_catchup_runs,
debug_crash_flags=None):
check.inst_param(instance, "instance", DagsterInstance)
check.inst_param(schedule_state, "schedule_state", ScheduleState)
check.inst_param(end_datetime_utc, "end_datetime_utc", datetime.datetime)
latest_tick = instance.get_latest_tick(schedule_state.schedule_origin_id)
if not latest_tick:
start_timestamp_utc = schedule_state.start_timestamp
elif latest_tick.status == ScheduleTickStatus.STARTED:
# Scheduler was interrupted while performing this tick, re-do it
start_timestamp_utc = latest_tick.timestamp
else:
start_timestamp_utc = latest_tick.timestamp + 1
start_datetime_utc = datetime.datetime.fromtimestamp(start_timestamp_utc,
tz=get_utc_timezone())
tick_times = list(
croniter_range(start_datetime_utc, end_datetime_utc,
schedule_state.cron_schedule))
if not tick_times:
logger.info("No new runs for {schedule_name}".format(
schedule_name=schedule_state.name))
return
if len(tick_times) > max_catchup_runs:
logger.warn(
"{schedule_name} has fallen behind, only launching {max_catchup_runs} runs"
.format(schedule_name=schedule_state.name,
max_catchup_runs=max_catchup_runs))
tick_times = tick_times[-max_catchup_runs:]
if len(tick_times) == 1:
logger.info("Launching run for {schedule_name} at {time}".format(
schedule_name=schedule_state.name,
time=tick_times[0].strftime(_SCHEDULER_DATETIME_FORMAT),
))
else:
logger.info(
"Launching {num_runs} runs for {schedule_name} at the following times: {times}"
.format(
num_runs=len(tick_times),
schedule_name=schedule_state.name,
times=", ".join([
time.strftime(_SCHEDULER_DATETIME_FORMAT)
for time in tick_times
]),
))
for schedule_time_utc in tick_times:
schedule_timestamp = get_timestamp_from_utc_datetime(schedule_time_utc)
if latest_tick and latest_tick.timestamp == schedule_timestamp:
tick = latest_tick
logger.info("Resuming previously interrupted schedule execution")
else:
tick = instance.create_schedule_tick(
ScheduleTickData(
schedule_origin_id=schedule_state.schedule_origin_id,
schedule_name=schedule_state.name,
timestamp=schedule_timestamp,
cron_schedule=schedule_state.cron_schedule,
status=ScheduleTickStatus.STARTED,
))
_check_for_debug_crash(debug_crash_flags, "TICK_CREATED")
with ScheduleTickHolder(tick, instance, logger) as tick_holder:
_check_for_debug_crash(debug_crash_flags, "TICK_HELD")
with RepositoryLocationHandle.create_from_repository_origin(
schedule_state.origin.repository_origin,
instance) as repo_location_handle:
repo_location = RepositoryLocation.from_handle(
repo_location_handle)
_schedule_run_at_time(
instance,
logger,
repo_location,
schedule_state,
schedule_time_utc,
tick_holder,
debug_crash_flags,
)
def start(interval=None,
log_to_file=True,
strage='local',
show_secret=False,
onebyone=False):
strages = ['local', 'travis', 'tencent']
if strage not in strages:
strage = 'local'
if log_to_file:
fh = logging.FileHandler('hostloc.log', mode='a')
fh.setLevel(logging.INFO)
fh.setFormatter(formatter)
logger.addHandler(fh)
if strage == 'tencent':
pass
else:
if show_secret:
logger.debug('本机IP: %s', get_ip())
else:
logger.debug('本机IP: %s', secret_log)
logger.debug('onebyone: %s', onebyone)
if onebyone:
user_length = len(accounts)
cron_match = None
with open('.github/workflows/hostloc.yml', 'r') as f:
cron_match = re.search(r"- cron: '(.*)'", f.read())
if cron_match:
cronsetting = cron_match.group(1)
now = datetime.datetime.now()
start_dt = datetime.datetime.combine(datetime.date.today(),
datetime.time.min)
end_dt = datetime.datetime.combine(datetime.date.today(),
datetime.time.max)
cronlist = list(croniter_range(start_dt, end_dt, cronsetting))
for i in enumerate(cronlist):
num, dt = i
if num == user_length:
break
elif now < dt:
logger.debug('正在执行第%s个任务...', num)
hostloc_checkin_retry(accounts[num - 1],
retry=3,
strage=strage,
show_secret=show_secret)
logger.debug('第%s个任务完成...', num)
break
elif dt == cronlist[-1]:
_first = True
_wait_time = interval or 5 * 60
for account in accounts[num:]:
if not _first:
logger.debug('等待%s分钟处理下一个任务', _wait_time // 60)
time.sleep(int(_wait_time))
_first = False
hostloc_checkin_retry(account,
retry=3,
strage=strage,
show_secret=show_secret)
logger.info('========= 今日任务完成 ==========')
else:
_first = True
_wait_time = interval or 5 * 60
for account in accounts:
if not _first:
logger.debug('等待%s分钟处理下一个任务', _wait_time // 60)
time.sleep(int(_wait_time))
_first = False
hostloc_checkin_retry(account,
retry=3,
strage=strage,
show_secret=show_secret)
logger.info('========= 今日任务完成 ==========')
