def kill(self, session=None):
"""关闭job ."""
job = session.query(BaseJob).filter(BaseJob.id == self.id).first()
job.end_date = timezone.system_now()
# 杀死job
try:
self.on_kill()
except Exception as e:
self.log.error('on_kill() method failed: {}'.format(e))
# 保存job的关闭时间
session.merge(job)
session.commit()
# 抛出异常
raise XToolException("Job shut down externally.")
def start(self):
"""创建一个文件处理子进程，
并将结果保存到self._result_queue结果队列
"""
thread_name = "{}_{}-Process".format(self.__class__.__name__,
self._instance_id)
self._process = multiprocessing.Process(target=self._handler,
args=(self._result_queue,
self.file_path,
self.args, self.kwargs),
name=thread_name)
self._process.start()
# 记录子进程启动时间
self._start_time = timezone.system_now()
return self._process
def heartbeat(self):
# 每次心跳获得最新的job状态
with create_session() as session:
# 如果只能查询到一个结果，返回它，否则抛出异常。
# 没有结果时抛sqlalchemy.orm.exc.NoResultFound，
# 有超过一个结果时抛sqlalchemy.orm.exc.MultipleResultsFound。
job = session.query(BaseJob).filter_by(id=self.id).one()
# 将job复制，并去掉与sesion的关联
# remove the association with any session
# and remove its “identity key”
make_transient(job)
session.commit()
# 如果job是关闭状态，则执行kill操作
if job.state == State.SHUTDOWN:
# 关闭job，抛出 AirflowException
self.kill()
# 获得到下一次心跳需要睡眠的时间间隔，并等待
sleep_for = 0
if job.latest_heartbeat:
sleep_for = max(
0, self.heartrate -
(timezone.system_now() - job.latest_heartbeat).total_seconds())
sleep(sleep_for)
# 睡眠之后会重新连接DB
with create_session() as session:
# 更新最新的心跳时间
job = session.query(BaseJob).filter(BaseJob.id == self.id).first()
job.latest_heartbeat = timezone.system_now()
session.merge(job)
session.commit()
# 执行心跳处理函数
self.heartbeat_callback(session=session)
self.log.debug('[heartbeat]')
def run(self):
"""新增一个job，并执行job，状态从运行态->完成态 ."""
with create_session() as session:
# 新增job
self.state = State.RUNNING
session.add(self)
session.commit()
id_ = self.id
make_transient(self)
self.id = id_
# 运行job
self._execute()
# job执行完成后，记录完成时间和状态
self.end_date = timezone.system_now()
self.state = State.SUCCESS
session.merge(self)
session.commit()
def __init__(self,
executor,
heartrate=5,
max_tis_per_query=512,
*args,
**kwargs):
# 当前机器的主机名
self.hostname = get_hostname()
# 执行器： 从任务队列中获取任务，并执行
self.executor = executor
# 获得执行器的类名
self.executor_class = executor.__class__.__name__
# 设置开始时间和心跳开始时间
self.start_date = timezone.system_now()
self.latest_heartbeat = self.start_date
# job的心跳间隔时间，用于判断job是否存活
self.heartrate = heartrate
# 调度器进程所在机器的执行用户
self.unixname = getpass.getuser()
# 因为需要批量更新任务实例的状态，为了防止SQL过长
# 需要设置每批更新的任务实例的数量
self.max_tis_per_query = max_tis_per_query
super(BaseJob, self).__init__(*args, **kwargs)
def heartbeat(self):
"""心跳
- 处理执行完毕的处理器
- 将任务加入队列
- 执行队列中的进程
"""
# 已完成的文件处理器
# :type : dict[unicode, AbstractFileProcessor]
finished_processors = {}
# 正在运行的文件处理器
# :type : dict[unicode, AbstractFileProcessor]
running_processors = {}
# 遍历所有的文件处理器
result = []
for file_path, processor in self._processors.items():
if processor.done:
self.log.info("Processor for %s finished", file_path)
# 获得已完成的文件处理器进程
finished_processors[file_path] = processor
# 文件处理器运行时间
now = timezone.system_now()
# 记录文件处理器的的执行时长
self._last_runtime[file_path] = (
now - processor.start_time).total_seconds()
# 记录文件处理器的结束时间
self._last_finish_time[file_path] = now
# 记录文件被处理的次数
self._run_count[file_path] += 1
# 收集已完成处理器的执行结果
if processor.result is None:
self.log.warning(
"Processor for %s exited with return code %s.",
processor.file_path, processor.exit_code)
else:
for value in processor.result:
result.append(value)
else:
# 记录正在运行的文件处理器进程
running_processors[file_path] = processor
# 每一次心跳，剔除已完成的处理器
self._processors = running_processors
self.log.debug("%s/%s scheduler processes running",
len(self._processors), self._parallelism)
self.log.debug("%s file paths queued for processing",
len(self._file_path_queue))
# 如果队列为空，设置需要入队的文件
if not self._file_path_queue:
# 可能存在正在运行的文件处理器进程尚未执行完成
# 在下一次心跳处理
file_paths_in_progress = self._processors.keys()
# 记录下尚未到调度时间的文件
file_paths_recently_processed = []
longest_parse_duration = 0
now = timezone.system_now()
# 遍历需要处理的文件列表
for file_path in self._file_paths:
# 获得文件处理器上一次执行完成的时间
last_finish_time = self.get_last_finish_time(file_path)
# 如果文件曾经处理过
if last_finish_time is not None:
duration = now - last_finish_time
# 如果文件尚未到调度时间，则记录在指定数组中
if duration.total_seconds() < self._process_file_interval:
file_paths_recently_processed.append(file_path)
# 获得所有文件中最长的等待时间
longest_parse_duration = max(duration.total_seconds(),
longest_parse_duration)
# 获得每次心跳的休眠时间
sleep_length = max(
self._min_file_parsing_loop_time - longest_parse_duration, 0)
# 休眠
if sleep_length > 0:
self.log.debug(
"Sleeping for %.2f seconds to prevent excessive "
"logging", sleep_length)
time.sleep(sleep_length)
# 获得处理器已经执行完成的文件列表，且这些文件的执行次数已经达到了最大阈值
files_paths_at_run_limit = [
file_path for file_path, num_runs in self._run_count.items()
if num_runs == self._max_runs
]
# 获得需要入队的文件，新增的文件在此入库
# 去掉正在运行的文件
# 去掉最近需要运行的文件
# 去掉运行次数已经达到阈值的文件
files_paths_to_queue = list(
set(self._file_paths) - set(file_paths_in_progress) -
set(file_paths_recently_processed) -
set(files_paths_at_run_limit))
# 遍历正在运行的处理器进程
for file_path, processor in self._processors.items():
self.log.debug(
"File path %s is still being processed (started: %s)",
processor.file_path, processor.start_time.isoformat())
self.log.debug("Queuing the following files for processing:\n\t%s",
"\n\t".join(files_paths_to_queue))
# 将任务加入队列
self._file_path_queue.extend(files_paths_to_queue)
# 处理器并发性阈值验证
while (self._parallelism - len(self._processors) > 0
and self._file_path_queue):
# 从队列中出队一个文件
file_path = self._file_path_queue.pop(0)
# 创建文件处理器子进程
processor = self._processor_factory(file_path)
# 启动文件处理器子进程
processor.start()
self.log.info(
"Started a process (PID: %s) to generate tasks for %s",
processor.pid, file_path)
# 记录文件子进程
self._processors[file_path] = processor
# 记录心跳的次数
self._run_count[self._heart_beat_key] += 1
# 返回已完成处理器的执行结果
return result
def get_runtime(self, file_path):
"""获得文件处理器的运行时长，单位是秒 ."""
if file_path in self._processors:
return (timezone.system_now() - self._processors[file_path].start_time)\
.total_seconds()
return None
def date_range(start_date, end_date=None, num=None, delta=None):
"""
Get a set of dates as a list based on a start, end and delta, delta
can be something that can be added to ``datetime.datetime``
or a cron expression as a ``str``
:param start_date: anchor date to start the series from
:type start_date: datetime.datetime
:param end_date: right boundary for the date range
:type end_date: datetime.datetime
:param num: alternatively to end_date, you can specify the number of
number of entries you want in the range. This number can be negative,
output will always be sorted regardless
:type num: int
>>> date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta=timedelta(1))
[datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0), datetime.datetime(2016, 1, 3, 0, 0)]
>>> date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta='0 0 * * *')
[datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0), datetime.datetime(2016, 1, 3, 0, 0)]
>>> date_range(datetime(2016, 1, 1), datetime(2016, 3, 3), delta="0 0 0 * *")
[datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 2, 1, 0, 0), datetime.datetime(2016, 3, 1, 0, 0)]
"""
if not delta:
return []
if end_date and start_date > end_date:
raise Exception("Wait. start_date needs to be before end_date")
if end_date and num:
raise Exception("Wait. Either specify end_date OR num")
# 结束时间默认为当前UTC时间
if not end_date and not num:
end_date = timezone.system_now()
delta_iscron = False
tz = start_date.tzinfo
if isinstance(delta, six.string_types):
delta_iscron = True
# 去掉开始时间的时区
if tz is not None:
start_date = timezone.make_naive(start_date, tz)
cron = croniter(delta, start_date)
elif isinstance(delta, timedelta):
delta = abs(delta)
l = []
if end_date:
while start_date <= end_date:
if timezone.is_naive(start_date):
l.append(timezone.make_aware(start_date, tz))
else:
l.append(start_date)
if delta_iscron:
start_date = cron.get_next(datetime)
else:
start_date += delta
else:
for _ in range(abs(num)):
if timezone.is_naive(start_date):
# 如果开始时间不存在时区信息，加上指定的时区信息
# 如果没有指定时区信息，默认系统时区
l.append(timezone.make_aware(start_date, tz))
else:
l.append(start_date)
if delta_iscron:
if num > 0:
start_date = cron.get_next(datetime)
else:
start_date = cron.get_prev(datetime)
else:
if num > 0:
start_date += delta
else:
start_date -= delta
return sorted(l)
def is_alive(self):
"""判断job是否存活 ."""
# 如果job超过2个心跳周期都没有上报心跳，则认为job已经死亡
return ((timezone.system_now() - self.latest_heartbeat).seconds <
(heartrate * 2.1))