def add_machine_info(machine_name, machine_ip):
'''
Add machine info record
@machin_name: machine name
@machin_ip: machine ip address
return True: sucess
False: fail
'''
# check has in mysql
row = select_machine_info(machine_name, machine_ip)
if len(row) != 0:
return True
# do insert
table = table_struct.T_MACHINE_INFO
cond_map = {
table_struct.MachineInfo.MachineName: machine_name,
table_struct.MachineInfo.MachineIP: machine_ip
}
mysql = mysql_wrapper.MysqlWrapper()
succ = mysql.insert(table, cond_map)
if succ is False:
logger.error('Insert machine info fail.')
return False
return True
def add_user_info(user_name, user_password_hash,user_email, register_time):
'''
Add user info record
@user_name: user name
@user_password_hash: user password hash
@user_email: user email
@register_time: register time
return True: sucess
False: fail
'''
# do insert
table = table_struct.T_USER_INFO
cond_map = {
table_struct.UserInfo.UserName: user_name,
table_struct.UserInfo.UserPasswordHash: user_password_hash,
table_struct.UserInfo.UserEmail: user_email,
table_struct.UserInfo.RegisterTime: register_time
}
mysql = mysql_wrapper.MysqlWrapper()
succ = mysql.insert(table, cond_map)
if succ is False:
logger.error('Insert user info fail.')
return False
return True
def add_task_info(cond_map):
'''
Add new task info record
@cond_map = {
'task_id': ...
'task_name': ...
...
}
return True/False
'''
table = table_struct.T_TASK_INFO
mysql = mysql_wrapper.MysqlWrapper()
# do select
rows = select_task_info(cond_map)
if len(rows) != 0:
logger.warn('Has task info in mysql.')
return True
# do insert
succ = mysql.insert(table, cond_map)
if succ is False:
logger.error('Add task info fail.')
return False
return True
def add_dag_run_history(cond_map):
'''
Add new dag run history record
@cond_map = {
'dag_id': ...
'dag_name': ...
...
}
return True/False
'''
table = table_struct.T_DAG_RUN_HISTORY
mysql = mysql_wrapper.MysqlWrapper()
# do select
rows = select_dag_run_history(cond_map)
if len(rows) != 0:
logger.warn('Has dag run history in mysql.')
return True
# do insert
succ = mysql.insert(table, cond_map)
if succ is False:
logger.error('Add dag run history fail.')
return False
return True
def select_max_user_id():
'''
Select max user id
return id/None
'''
table = table_struct.T_USER_INFO
# do select
mysql = mysql_wrapper.MysqlWrapper()
rows = mysql.select(table, cond_map)
if len(rows) == 0:
logger.error('Select user info get empty rows')
return dict()
return rows[0]
def select_all_task_run_history():
'''
Select all task run history record
return rows/list()
'''
table = table_struct.T_TASK_RUN_HISTORY
condition_str = '1=1'
mysql = mysql_wrapper.MysqlWrapper()
rows = mysql.select_with_condition(table, condition_str)
if len(rows) == 0:
logger.error('Select all task run history get empty rows')
return list()
return rows
def select_all_task_info():
'''
Select all task info record
return rows/list()
'''
table = table_struct.T_TASK_INFO
condition_str = '1=1'
mysql = mysql_wrapper.MysqlWrapper()
rows = mysql.select_with_condition(table, condition_str)
if len(rows) == 0:
logger.error('Select all task info get empty rows')
return list()
return rows
def select_need_start_dag(current_time):
'''
Select all need start dag info
return rows/list()
'''
table = table_struct.T_DAG_INFO
mysql = mysql_wrapper.MysqlWrapper()
condition_str = '(%s = 1 and %s != \'%s\' and %s != \'%s\' and %s <= \'%s\') \
or (%s = 1 and %s = \'%s\' and %s = 1 and %s <= \'%s\')'% \
(table_struct.DagInfo.Valid, able_struct.DagInfo.DagStatus, status.DAG_RUNNING, \
table_struct.DagInfo.DagStatus, status.DAG_FAILED, table_struct.DagInfo.NextStartTime, \
current_time, table_struct.DagInfo.Valid, table_struct.DagInfo.DagStatus, \
status.DAG_FAILED, table_struct.DagInfo.SkipFailed, table_struct.DagInfo.NextStartTime, current_time)
return select_dag_info_with_condition(condition_str)
def select_max_user_id():
'''
Select max user id
return id/0
'''
table = table_struct.T_USER_INFO
cond_str = '1=1'
fields = table_struct.UserInfo.ID
order_field = table_struct.UserInfo.ID
order_desc = True
mysql = mysql_wrapper.MysqlWrapper()
rows = mysql.select_with_condition(table, cond_str, fields, order_field, order_desc)
if len(rows) == 0:
return 0
return rows[0][table_struct.UserInfo.ID]
def select_dag_info_with_condition(cond_str, fields = '*'):
'''
With by condition string select dag info record
@cond_str: condition string
@fields: select fields, default equal *
return rows/list()
'''
table = table_struct.T_DAG_INFO
# do select
mysql = mysql_wrapper.MysqlWrapper()
rows = mysql.select_with_condition(table, cond_str, fields)
if len(rows) == 0:
logger.error('Select dag info get empty rows')
return list()
return rows
def select_pending_tasks(machine_ip, fields='*'):
'''
With by condition map select pending tasks record
@machine_ip: run machine ip address
@fields: select fields, default equal *
return rows/list()
'''
table = table_struct.T_TASK_PENDING_QUEUE
mysql = mysql_wrapper.MysqlWrapper()
cond_map = {table_struct.TaskPendingQueue.MachineIp: machine_ip}
rows = mysql.select(table, cond_map, fields)
if len(rows) == 0:
logger.error('Select pending tasks get empty rows')
return list()
return rows
def delete_project_info(cond_map):
'''
delete project info record
@cond_map = {
'project_id': ...
...
}
return True/False
'''
table = table_struct.T_PROJECT_INFO
mysql = mysql_wrapper.MysqlWrapper()
succ = mysql.delete(table, cond_map)
if succ is False:
logger.error('Delete project info fail.')
return False
return True
def select_task_by_id(task_id, fields = '*'):
'''
Select task by task id
@task_id: task id
return row/list()
'''
table = table_struct.T_TASK_INFO
mysql = mysql_wrapper.MysqlWrapper()
cond_map = {
table_struct.TaskInfo.TaskId: task_id
}
rows = mysql.select(table, cond_map, fields)
if len(rows) == 0:
logger.error('Select task info get empty rows')
return dict()
return rows[0]
def delete_task_info(cond_map):
'''
delete task info record
@cond_map = {
'task_id': ...
'task_name': ...
...
}
return True/False
'''
table = table_struct.T_TASK_INFO
mysql = mysql_wrapper.MysqlWrapper()
succ = mysql.delete(table, cond_map)
if succ is False:
logger.error('Delete task info fail.')
return False
return True
def delete_task_run_history(cond_map):
'''
delete task run history record
@cond_map = {
'task_id': ...
'task_name': ...
...
}
return True/False
'''
table = table_struct.T_TASK_RUN_HISTORY
mysql = mysql_wrapper.MysqlWrapper()
succ = mysql.delete(table, cond_map)
if succ is False:
logger.error('Delete task run history fail.')
return False
return True
def select_dag_running_task(dag_id):
'''
select dag all running task
@dag_id: dag id
return rows/list()
'''
table = table_struct.T_TASK_INFO
mysql = mysql_wrapper.MysqlWrapper()
cond_map = {
table_struct.TaskInfo.DagId: dag_id,
table_struct.TaskInfo.TaskStatus: status.TASK_RUNNING
}
rows = mysql.select(table, cond_map, fields)
if len(rows) == 0:
logger.error('dag:[%d] select running tasks get empty.' % dag_id)
return list()
return rows
def select_project_info(cond_map, fields='*'):
'''
With by condition map select project info record
@cond_map = {
'project_id': ...
...
}
@fields: select fields, default equal *
return rows/list()
'''
table = table_struct.T_PROJECT_INFO
# do select
mysql = mysql_wrapper.MysqlWrapper()
rows = mysql.select(table, cond_map, fields)
if len(rows) == 0:
logger.error('Select proejct info get empty rows')
return list()
return rows
def delete_machine_info(cond_map):
'''
delete machine info record
@cond_map = {
user_id: xxx
...
}
return True: sucess
False: fail
'''
# do delete record
table = table_struct.T_MACHINE_INFO
mysql = mysql_wrapper.MysqlWrapper()
succ = mysql.delete(table, cond_map)
if succ is False:
logger.error('Delete machine info fail.')
return False
return True
def add_pending_task(task_id, machine_ip):
'''
Add new pending task record
@task_id: task id
@machine_ip: machine ip address
return True/False
'''
table = table_struct.T_TASK_PENDING_QUEUE
mysql = mysql_wrapper.MysqlWrapper()
cond_map = {
table_struct.TaskPendingQueue.TaskId: task_id,
table_struct.TaskPendingQueue.MachineIp: machine_ip
}
succ = mysql.insert(table, cond_map)
if succ is False:
logger.error('Add new pending task fail.')
return False
return True
def delete_pending_task(task_id, machine_ip):
'''
delete pending task record
@task_id: ...
@machine_ip: ...
return True/False
'''
table = table_struct.T_TASK_PENDING_QUEUE
mysql = mysql_wrapper.MysqlWrapper()
cond_map = {
table_struct.TaskPendingQueue.TaskId: task_id,
table_struct.TaskPendingQueue.MachineIp: machine_ip
}
succ = mysql.delete(table, cond_map)
if succ is False:
logger.error('Delete pending task fail.')
return False
return True
def select_task_run_history(cond_map, fields='*'):
'''
With by condition map select task run history record
@cond_map = {
'task_id': ...
'task_name': ...
...
}
@fields: select fields, default equal *
return rows/list()
'''
table = table_struct.T_TASK_RUN_HISTORY
# do select
mysql = mysql_wrapper.MysqlWrapper()
rows = mysql.select(table, cond_map, fields)
if len(rows) == 0:
logger.error('Select task run history get empty rows')
return list()
return rows
def select_user_info(cond_map):
'''
Select user info record
cond_map = {
user_name: user name
user_email: user email
...
}
return row: success
[]: fail
'''
table = table_struct.T_USER_INFO
# do select
mysql = mysql_wrapper.MysqlWrapper()
rows = mysql.select(table, cond_map)
if len(rows) == 0:
logger.error('Select user info get empty rows')
return dict()
return rows[0]
def select_machine_info(machine_name, machine_ip):
'''
Select machine info record
@machin_name: machine name
@machin_ip: machine ip address
return rows: success
[]: fail
'''
table = table_struct.T_MACHINE_INFO
cond_map = {
table_struct.MachineInfo.MachineName: machine_name,
table_struct.MachineInfo.MachineIP: machine_ip
}
# do select
mysql = mysql_wrapper.MysqlWrapper()
rows = mysql.select(table, cond_map)
if len(rows) == 0:
logger.error('Select machine info get empty rows')
return dict()
return rows[0]
def update_project_info(new_value_map, cond_map):
'''
update project info record
@new_value_map = {
'project_id': ...
...
}
@cond_map = {
'project_id': ...
...
}
return True/False
'''
table = table_struct.T_PROJECT_INFO
mysql = mysql_wrapper.MysqlWrapper()
succ = mysql.update(table, new_value_map, cond_map)
if succ is False:
logger.error('Update project info fail.')
return False
return True
def update_task_info(new_value_map, cond_map):
'''
update task info record
@new_value_map = {
'task_id': ...
'task_name': ...
...
}
@cond_map = {
'task_id': ...
'task_name': ...
...
}
return True/False
'''
table = table_struct.T_TASK_INFO
mysql = mysql_wrapper.MysqlWrapper()
succ = mysql.update(table, new_value_map, cond_map)
if succ is False:
logger.error('Update task info fail.')
return False
return True
def add_project_info(cond_map):
'''
Add new project info record
@cond_map = {
'project_id': ...
...
}
return True/False
'''
table = table_struct.T_PROJECT_INFO
mysql = mysql_wrapper.MysqlWrapper()
# do select
rows = select_project_info(cond_map)
if len(rows) != 0:
logger.warn('Has proejct info in mysql.')
return True
# do insert
succ = mysql.insert(table, cond_map)
if succ is False:
logger.error('Add proejct info fail.')
return False
return True
def update_machine_info(new_value_map, cond_map):
'''
udpate machine info record
@new_value_map = {
'machine_name': ...
'machine_ip': ...
}
@cond_map = {
'machine_name': ...
'machine_ip': ...
}
return True: sucess
False: fail
'''
# do update record
table = table_struct.T_MACHINE_INFO
mysql = mysql_wrapper.MysqlWrapper()
succ = mysql.update(table, new_value_map, cond_map)
if succ is False:
logger.error('Update machine info fail.')
return False
return True
def update_task_run_history(new_value_map, cond_map):
'''
update task run history record
@new_value_map = {
'task_id': ...
'task_name': ...
...
}
@cond_map = {
'task_id': ...
'task_name': ...
...
}
return True/False
'''
table = table_struct.T_TASK_RUN_HISTORY
mysql = mysql_wrapper.MysqlWrapper()
succ = mysql.update(table, new_value_map, cond_map)
if succ is False:
logger.error('Update task run history fail.')
return False
return True
def __init__(self):
self.db_wrapper = mysql_wrapper.MysqlWrapper()
self.conf = database_config.DatabaseConfig()
def __init__(self):
self.db = mysql_wrapper.MysqlWrapper()
