def get_count_query(self):
if wwwutils.get_filter_by_user():
curr_user = airflow.login.current_user
return self.session.query(func.count('*')).filter(
self.model.username == curr_user.user.username)
else:
return self.session.query(func.count('*'))
def get_query(self):
if wwwutils.get_filter_by_user():
curr_user = airflow.login.current_user
return self.session.query(self.model).filter(
self.model.is_subdag == False,
self.model.owners == curr_user.user.username)
else:
return self.session.query(
self.model).filter(self.model.is_subdag == False)
def index(self, session=None):
# if request.method == 'POST':
# request_data = request.get_json(force=True).get('uri')
# 参数
# dagid, execution_date, num_days , display
dagid = request.args.get('dagid')
execution_date = request.args.get('execution_date')
num_days = request.args.get('num_days')
display = request.args.get('display') # day,day+dag,dag+dag+scheduler
sort = request.args.get('sort')
state = request.args.get('state')
#dagid = 'dag008'
if (not dagid) or len(dagid) < 0:
dagid = ''
if (not num_days):
num_days = 1
else:
num_days = int(num_days)
if (not execution_date) or len(execution_date) < 0:
execution_date = date.today().isoformat()
if (not display) or len(display) < 0:
display = 'day+dag'
if (not sort) or len(sort) < 0:
sort = 'time'
if (not state) or len(state) < 0:
state = 'all'
curr_user = airflow.login.current_user
# 是否为超级用户，普通用户只能看到自己
dag_filter_sql = ""
if wwwutils.get_filter_by_user():
dag_filter_sql = " ( dag_id IN ( SELECT dag_name FROM dcmp_dag WHERE last_editor_user_id = %s ) ) " % curr_user.user.id
else:
dag_filter_sql = " (1=1) "
# dagid 搜索条件
dagid_search_sql = ""
if (dagid) and len(dagid) > 0 and dagid.find(',') >= 0:
dagid_search_sql = " ('," + dagid + ",'" + " like concat( '%,',dag_id,',%' ) ) "
elif (dagid) and len(dagid) > 0 and dagid.find(',') < 0:
dagid_search_sql = " ( dag_id like concat( '%','" + dagid + "','%' ) ) "
else:
dagid_search_sql = " (1=1) "
state_search_sql = ""
if state == 'all':
state_search_sql = " (1=1) "
else:
state_search_sql = " ( state = '" + state + "' ) "
if sort == 'time':
sortsql = ' start_date asc ,execution_date asc ,dag_id asc'
elif sort == 'dag':
sortsql = ' dag_id asc,execution_date asc ,start_date asc '
else:
sortsql = ' TIMESTAMPDIFF( SECOND, start_date , IFNULL(end_date,NOW()) ) desc '
session = settings.Session()
mysql_query = """
SELECT
`id`,
`dag_id`,
`execution_date`,
`state`,
`run_id`,
IFNULL(end_date,NOW()) end_date,
`start_date`,
DATE_FORMAT(start_date, '%%Y-%%m-%%d') execution_dt ,
CAST( CONCAT('2018-01-01 ', DATE_FORMAT(start_date, '%%H:%%i:%%s')) AS DATETIME ) as start_date_display ,
DATE_ADD( CAST( CONCAT('2018-01-01 ', DATE_FORMAT(start_date, '%%H:%%i:%%s')) AS DATETIME )
, INTERVAL TIMESTAMPDIFF( SECOND, start_date , IFNULL(end_date,NOW())) SECOND) end_date_display
FROM
`dag_run`
WHERE start_date >= DATE_ADD(:execution_date,INTERVAL :start_days DAY )
AND start_date < DATE_ADD(:execution_date,INTERVAL :end_days DAY )
AND %s
AND %s
AND %s
ORDER BY %s
""" % (dagid_search_sql, dag_filter_sql,
state_search_sql, sortsql)
result = session.execute(
mysql_query,
{
'execution_date': execution_date,
'start_days': -1 * (num_days - 1),
'end_days': 1 # MySQLdb.escape_string(dagidsql)
})
#logging.warning('RESULT-FROM-QUERY:')
if result.rowcount > 0:
records = result.fetchall()
# gantt chart data
tasks = []
taskNames = []
tasksUniq = set() # set
# 修改图形距离左侧的，重写gantt-chart-d3v2.js的yAxisLeftOffset function
if display == 'day':
for dagInst in records:
tasks.append({
'startDate':
wwwutils.epoch(dagInst.start_date_display),
'endDate':
wwwutils.epoch(dagInst.end_date_display),
'isoStart':
dagInst.start_date.isoformat()[:-7],
'isoEnd':
dagInst.end_date.isoformat()[:-7],
'taskName':
dagInst.execution_dt,
'duration':
"{}".format(dagInst.end_date -
dagInst.start_date)[:-7],
'status':
dagInst.state,
'runId':
dagInst.run_id,
'runName':
dagInst.dag_id,
'executionDate':
dagInst.execution_date.isoformat(),
})
taskNames = [dagInst.execution_dt for dagInst in records]
elif display == 'day+dag':
for dagInst in records:
tasks.append({
'startDate':
wwwutils.epoch(dagInst.start_date_display),
'endDate':
wwwutils.epoch(dagInst.end_date_display),
'isoStart':
dagInst.start_date.isoformat()[:-7],
'isoEnd':
dagInst.end_date.isoformat()[:-7],
'taskName':
dagInst.execution_dt + '[' + dagInst.dag_id + ']',
'duration':
"{}".format(dagInst.end_date -
dagInst.start_date)[:-7],
'status':
dagInst.state,
'runId':
dagInst.run_id,
'runName':
dagInst.dag_id,
'executionDate':
dagInst.execution_date.isoformat(),
})
taskNames = [
dagInst.execution_dt + '[' + dagInst.dag_id + ']'
for dagInst in records
]
elif (display == 'dag+scheduler(1day)'
and num_days == 1): # dag+dag+scheduler
for dagInst in records:
tasks.append({
'startDate':
wwwutils.epoch(dagInst.start_date_display),
'endDate':
wwwutils.epoch(dagInst.end_date_display),
'isoStart':
dagInst.start_date.isoformat()[:-7],
'isoEnd':
dagInst.end_date.isoformat()[:-7],
'taskName':
dagInst.dag_id + '[' + dagInst.run_id + ']',
'duration':
"{}".format(dagInst.end_date -
dagInst.start_date)[:-7],
'status':
dagInst.state,
'runId':
dagInst.run_id,
'runName':
dagInst.dag_id,
'executionDate':
dagInst.execution_date.isoformat(),
})
taskNames = [
dagInst.dag_id + '[' + dagInst.run_id + ']'
for dagInst in records
]
yAxisLeftOffset = 0
for ta in tasks:
tasksUniq.add(ta['taskName'])
if yAxisLeftOffset < len(ta['taskName']):
yAxisLeftOffset = len(ta['taskName'])
# 修改横条的总高度
if len(tasksUniq) < 2:
height = len(tasksUniq) * 45 + 35
elif len(tasksUniq) < 3:
height = len(tasksUniq) * 45 + 25
elif len(tasksUniq) < 4:
height = len(tasksUniq) * 45 + 15
else:
height = len(tasksUniq) * 45
states = {dagInst.state: dagInst.state for dagInst in records}
data = {
'taskNames': taskNames,
'tasks': tasks,
'taskStatus': states,
'yAxisLeftOffset': yAxisLeftOffset * 6,
'height': height,
}
else:
logging.warning('No records found')
#dag_runs = []
data = []
#dttm = datetime.now().date()
#form = DateTimeForm(data={'execution_date': dttm})
logging.warning('##########################1')
logging.warning(mysql_query)
#logging.warning('##########################2')
#logging.warning(tasksUniq)
#logging.warning('##########################3')
#logging.warning(data)
#logging.warning('##########################4')
#logging.warning(tasks)
#logging.warning('##########################5')
#logging.warning(records)
logging.warning('##########################6')
return self.render(
"dags-charts/dagrunninghischart.html"
# , dag_runs=json.dumps(dag_runs)
,
data=json.dumps(data, indent=2)
# , execution_date=datetime.now().isoformat()
# , form= form
,
execution_date=execution_date,
num_days=num_days,
dagid=dagid,
display=display,
sort=sort,
state=state,
dag='')
def index(self, session=None):
TASK_NAME = "Task Name"
COMMAND = "Command"
request_args_filter = RequestArgsFilter(DcmpDag, request.args, (
("Category", {
"operations": ["contains"]
}),
(TASK_NAME, {
"operations": ["contains"],
"no_filters": True
}),
(COMMAND, {
"operations": ["contains"],
"no_filters": True
}),
))
confs = OrderedDict()
current_user = get_current_user()
curr_user = airflow.login.current_user
do_filter = FILTER_BY_OWNER and (not curr_user.is_superuser())
owner_mode = conf.get('webserver', 'OWNER_MODE').strip().lower()
if wwwutils.get_filter_by_user():
dcmp_dags = session.query(DcmpDag).order_by(
DcmpDag.dag_name).filter(
DcmpDag.last_editor_user_name == curr_user.user.username,
*request_args_filter.filters)
else:
dcmp_dags = session.query(DcmpDag).order_by(
DcmpDag.dag_name).filter(*request_args_filter.filters)
dcmp_dags_count = dcmp_dags.count()
dcmp_dags = dcmp_dags[:]
for dcmp_dag in dcmp_dags:
dcmp_dag.conf = dcmp_dag.get_conf(session=session)
if request_args_filter.filters_dict.get(TASK_NAME):
task_name_value = request_args_filter.filters_dict.get(
TASK_NAME)["value"]
def filter_dcmp_dags_by_task_name(dcmp_dag):
for task in dcmp_dag.conf["tasks"]:
if task_name_value in task["task_name"]:
return True
return False
dcmp_dags = filter(filter_dcmp_dags_by_task_name, dcmp_dags)
if request_args_filter.filters_dict.get(COMMAND):
command_value = request_args_filter.filters_dict.get(
COMMAND)["value"]
def filter_dcmp_dags_by_command(dcmp_dag):
for task in dcmp_dag.conf["tasks"]:
if command_value in task["command"]:
return True
return False
dcmp_dags = filter(filter_dcmp_dags_by_command, dcmp_dags)
search = request.args.get("search", "")
if search:
searched_dcmp_dags = []
for dcmp_dag in dcmp_dags:
dcmp_dag.search_results = []
for result_task_name, result_key, result_line in search_conf_iter(
search, dcmp_dag.conf):
dcmp_dag.search_results.append({
"key":
result_key,
"full_key":
"%s__%s" % (result_task_name, result_key),
"line":
result_line,
"html_line": ('<span class="nb">[%s]</span> ' %
result_key if result_key else "") +
result_line.replace(
search,
'<span class="highlighted">%s</span>' % search),
})
if dcmp_dag.search_results:
searched_dcmp_dags.append(dcmp_dag)
dcmp_dags = searched_dcmp_dags
return self.render(
"dcmp/index.html",
can_access_approver=can_access_approver(),
dcmp_dags=dcmp_dags,
dcmp_dags_count=dcmp_dags_count,
filter_groups=request_args_filter.filter_groups,
active_filters=request_args_filter.active_filters,
search=search,
)