def _get_waitforsend_task(self) -> iter:
try:
for task in self._dbmanager.get_deliverable_iscouttask():
task: IscoutTask = task
if not isinstance(task, IscoutTask):
self._logger.warn(
"Invalid IscoutTask from database for delivering: {}".
format(task))
continue
# 判断指定的开始时间
if (isinstance(task.cmd.stratagy.time_start, str)
and not task.cmd.stratagy.time_start == ""):
# 这里cmd里来的就是东8区时间
# tstart: float = time.mktime(
# time.strptime(task.cmd.stratagy.time_start,
# "%Y-%m-%d %H:%M:%S"))
tstart = parser.parse(task.cmd.stratagy.time_start)
tstart = tstart.timestamp()
if tstart > helper_time.ts_since_1970_tz():
continue
yield task
except Exception:
self._logger.error("Get wait for send task error: {}".format(
traceback.format_exc()))
def _filter_deliverable_task(self, task: IscoutTask) -> bool:
res: bool = False
try:
# 这里task.lastendtime就是东8区时间
tend = task.lastendtime
if tend is None or tend == "":
return res
if task.cmd.stratagy.interval is None:
raise Exception(
f"Period task interval is None: taskid={task.taskid}")
tstart = tend + task.cmd.stratagy.interval
tnow = helper_time.ts_since_1970_tz()
if tnow < tstart:
return res
# 走到这，就是周期任务达到 循环间隔，需要下发
res = True
# 只有在周期任务下发的时候才+1其他任何时候都不增加周期，不然就会出错
task.periodnum += 1
except Exception:
self._logger.error(
"Check if task is deliverable error:\ntaskid={}\nbatchid={}\nerror:{}"
.format(task.taskid, task.batchid, traceback.format_exc()))
return res
def _deal_equal_period(self, tb: AutotaskBatchBack,
task: AutomatedTask) -> bool:
res: bool = False
try:
# 查询对应的总任务对象
# 子任务是这些状态，那么总任务的 batchcompletecount+=1
# 把当前AutotaskBatchBack子任务查出来
if tb._cmdstatus==ECommandStatus.Failed or \
tb._cmdstatus==ECommandStatus.Succeed or \
tb._cmdstatus==ECommandStatus.Timeout or \
tb._cmdstatus==ECommandStatus.Cancelled:
tb.progress = 1
if not self._dbmanager.is_autobtask_batchcompletecount_increaced(
tb._platform, tb.taskid, tb.batchid):
if not self._dbmanager.increace_autotask_batch_complete_count(
tb._platform, tb.taskid, tb.batchid):
tb._cmdrcvmsg = "全自动子任务完成数量增加到数据库失败"
self._logger.error(
"Increace AutomatedTask batch complete count failed:\ndata:{}\ntaskid={}\nbatchid={}\nperiodnum={}\nbatchtask_cmdstatus={}"
.format(tb.inputdata.name, tb.taskid, tb.batchid,
tb.periodnum, tb._cmdstatus.name))
return res
elif not task.isbatchcompletecountincreased:
task.batchcompletecount += 1
task.isbatchcompletecountincreased = True
# PeriodNum=?,
# Status=?,
# Progress=?,
# CmdRcvMsg=?,
# Sequence=?,
# UpdateTime=?
if not self._dbmanager.update_automated_batch_task(
tb._platform,
tb.taskid,
tb.batchid,
updatefields={
# 'PeriodNum': tb.periodnum,
'Status': tb.cmdstatus.value,
'Progress': 1,
'CmdRcvMsg': tb.cmdrcvmsg,
'Sequence': tb.sequence,
'UpdateTime': tb.time,
'LastEndTime': helper_time.ts_since_1970_tz(),
}):
tb._cmdrcvmsg = "更新Autom子任务状态到本地数据库失败"
self._logger.error(
"Update AutotaskBatchBack to db failed:\ndata:{}\ntaskid:{}\nbatchid:{}"
.format(tb.inputdata.name, tb.taskid, tb.batchid))
return res
res = True
except Exception:
self._logger.error(
"Deal first period task back error:\ntaskid:{}\nbatchid:{}\nclientid:{}\nerror:{}"
.format(task.taskid, task.batchid, task._clientid,
traceback.format_exc()))
return res
def _output_taskback(self, task: AutomatedTask, status: ECommandStatus,
msg: str):
if not isinstance(task, AutomatedTask):
self._logger.error(
"Invalid AutomatedTask object for output AutoTaskBack: {}".
format(type(task)))
return
task: AutomatedTask = task
taskback: AutotaskBack = AutotaskBack.create_from_task(
task, status, msg, helper_time.ts_since_1970_tz())
if not OutputManagement.output(taskback):
self._logger.error("Output AutoTaskBack failed: taskid={}".format(
task.taskid))
def _add_required_fields(self, seg: DataSeg, data: InputData):
"""向DataSeg中添加当前转换器配置的必要字段"""
# 添加额外字段
if not self._extendfields is None:
for ef in self._extendfields.items():
if not seg.contains_key(ef[0]):
if not callable(ef[1]):
seg.append_to_fields(ef[0], ef[1])
else:
seg.append_to_fields(ef[0], ef[1]())
# 添加额外必要字段
if not seg.contains_key("platform"):
seg.append_to_fields("platform", data._platform)
# 新任务过来，要加一个time
if not seg.contains_key("time"):
seg.append_to_fields("time", helper_time.ts_since_1970_tz())
def _deal_equal_period(self, tb: IscanTaskBack, task: IscanTask) -> bool:
res: bool = False
try:
# 查询对应的总任务对象
# 子任务是这些状态，那么总任务的 batchcompletecount+=1
# 把当前IScouTBatchTask子任务查出来
updatefields: dict = {}
if tb._cmdstatus == ECommandStatus.Failed or \
tb._cmdstatus == ECommandStatus.Succeed or \
tb._cmdstatus == ECommandStatus.Timeout or \
tb._cmdstatus == ECommandStatus.Cancelled:
# 周期数+1
# if task._is_period:
# tb.periodnum += 1
# task.periodnum += 1
tb.progress = 1
updatefields["LastEndTime"] = helper_time.ts_since_1970_tz()
updatefields["PeriodNum"] = tb.periodnum
updatefields["Status"] = tb.cmdstatus.value
updatefields["Progress"] = tb.progress
updatefields["CmdRcvMsg"] = tb.cmdrcvmsg
updatefields["Sequence"] = tb._sequence
updatefields["UpdateTime"] = tb.time
if not self._dbmanager.update_iscantask2(
tb._platform,
tb._taskid,
updatefields=updatefields,
):
tb._cmdrcvmsg = "更新IScout子任务状态到本地数据库失败"
self._logger.error(
"Update IScanTask to db failed:\ndata:{}\ntaskid:{}\n".
format(tb.inputdata.name, tb._taskid))
return res
res = True
except Exception:
self._logger.error(
"Deal first period task back error:\ntaskid:{}\nbatchid:{}\nclientid:{}\nerror:{}"
.format(task.taskid, task.batchid, task._clientid,
traceback.format_exc()))
return res
def _dispatch_to_all_clients(self, cmd: IdownCmd) -> (bool, str):
'''dispatch to all clients'''
succ: bool = True
msg: str = None
try:
currentclients: dict = StatusMantainer.get_clientstatus_id_sorted()
# 每个ClientId存一份cmd
if not self._dbmanager.save_new_idown_cmd(
cmd.platform, currentclients, cmd,
helper_time.ts_since_1970_tz(), ECommandStatus.WaitForSend,
None, None, 0):
succ = False
msg = '分配命令到所有采集端失败'
except Exception as ex:
self._logger.error(
"Dispatch IDownCmd to all clients error: {}".format(ex.args))
succ = False
msg = "内部错误: {}".format(ex.args)
return (succ, msg)
def _dispatch_by_target(self, cmd: IdownCmd) -> (bool, str):
'''dispatch by cmd target'''
# cmd.target.
succ: bool = True
msg: str = None
try:
if not isinstance(cmd, IdownCmd):
self._logger.error(
"Invalid IDownCmd for dispatch by target: {}".format(
type(cmd)))
return (succ, msg)
task: Task = self._dbmanager.search_task_by_cmd_target(cmd)
if not isinstance(task, Task):
succ = False
msg = '未找到指定的target'
return (succ, msg)
if not isinstance(task._clientid, str):
succ = False
msg = '指定的target任务未下发到任何采集端'
return (succ, msg)
currentclients: dict = StatusMantainer.get_clientstatus_id_sorted()
if not currentclients.__contains__(task._clientid):
succ = False
msg = '指定的target任务所在的采集端未上线'
return (succ, msg)
clients: dict = {task._clientid: currentclients[task._clientid]}
if not self._dbmanager.save_new_idown_cmd(
cmd.platform, clients, cmd, helper_time.ts_since_1970_tz(),
ECommandStatus.WaitForSend, None, None, 0):
succ = False
msg = '分配命令到所有采集端失败'
except Exception as ex:
succ = False
msg = "内部错误: {}".format(ex.args)
return (succ, msg)
def _get_waitforsend_task(self) -> iter:
try:
for task in self._dbmanager.get_deliverable_autotask():
task: AutomatedTask = task
if not isinstance(task, AutomatedTask):
self._logger.warn(
"Invalid AutomatedTask from database for delivering: {}"
.format(task))
continue
# 判断指定的开始时间
if isinstance(task.cmd.stratagy.time_start,
str) and not task.cmd.stratagy.time_start == "":
# 这个cmd里来的time_start就是东8区时间
tstart: float = time.mktime(
time.strptime(task.cmd.stratagy.time_start,
"%Y-%m-%d %H:%M:%S"))
if tstart > helper_time.ts_since_1970_tz():
continue
yield task
except Exception:
self._logger.error("Get wait for send task error: {}".format(
traceback.format_exc()))
class TbIDownCmd(TbSqliteBase):
"""Task表及相关操作"""
__tb_IDownCmd: SqliteTable = SqliteTable(
'IDownCmd',
True,
SqliteColumn(colname='Id',
coltype='INTEGER',
nullable=False,
is_primary_key=True,
is_auto_increament=True,
is_unique=True).set_index_new(),
SqliteColumn(colname='ClientId').set_index_new(),
SqliteColumn(colname='CmdId', nullable=False).set_index_new(),
SqliteColumn(colname='Platform', nullable=False).set_index_new(),
SqliteColumn(colname='TaskId').set_index_new(),
SqliteColumn(colname='BatchId').set_index_new(),
SqliteColumn(colname='Cmd', nullable=False).set_index_new(),
SqliteColumn(colname='Status', coltype='INTEGER',
nullable=False).set_index_new(),
SqliteColumn(colname='Source').set_index_new(),
SqliteColumn(colname='CmdRcvMsg'),
SqliteColumn(colname='CreateTime',
coltype='DATETIME',
defaultval=helper_time.ts_since_1970_tz()),
SqliteColumn(colname='UpdateTime', coltype='REAL',
nullable=False).set_index_new(),
SqliteColumn(colname='Sequence',
nullable=False,
coltype='INTEGER',
defaultval=0).set_index_new(),
SqliteColumn(
# 指示当前cmd是否是与任何task关联的，0独立的/1关联的
colname='IsRelative',
nullable=False,
coltype='INTEGER',
defaultval=1).set_index_new(),
)
# 主键 ClientId+CmdId
# 所有列，复制粘贴用...：
# ClientId
# CmdId,
# Platform,
# TaskId,
# BatchId,
# Cmd,
# Status,
# Source,
# CmdRcvMsg,
# CreateTime,
# UpdateTime,
# Sequence,
# IsRelative
def __init__(self, dbcfg: SqliteConfig):
""""""
TbSqliteBase.__init__(self, TbIDownCmd.__tb_IDownCmd._tbname, dbcfg)
def _append_tables(self):
self._conn_mngr.append_table(TbIDownCmd.__tb_IDownCmd)
@table_locker(__tb_IDownCmd._tbname)
def save_new_idowncmd(
self,
platform: str,
client: Client,
command: IdownCmd,
cmdtime: float,
cmdstatus: ECommandStatus = ECommandStatus.WaitForSend,
taskid: str = None,
batchid: str = None,
isrelative: int = 1,
) -> bool:
"""保存新的批处理任务的令牌资源\n
task:任务对象"""
res = False
conn: SqliteConn = None
cursor = None
client: Client = client
command: IdownCmd = command
try:
# 搜索每个库，看有没有 TokenId一样的，且时间更新
# 的，一样就更新其他所有字段
cmd = f'''SELECT COUNT(1) FROM {self._tbname} WHERE
ClientId=? and CmdId=? and Platform=?'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
cursor.execute(cmd, (
client._statusbasic._clientid,
command.cmd_id,
platform,
))
result = cursor.fetchall()
if result[0][0] > 0:
res = True
cmd = f'''UPDATE {self._tbname} set
Cmd=?,
TaskId=?,
BatchId=?,
Status=?,
Source=?,
CreateTime=?,
UpdateTime=?,
IsRelative=?
WHERE ClientId=? and CmdId=? and Platform=? and CreateTime<=?;'''
result = cursor.execute(cmd, (
command.cmd_str,
taskid,
batchid,
cmdstatus.value,
command.source,
helper_time.timespan_to_datestr_tz(cmdtime),
helper_time.get_time_sec_tz(),
isrelative,
client._statusbasic._clientid,
command.cmd_id,
platform,
helper_time.timespan_to_datestr_tz(cmdtime),
))
# 这句没用，就是调试看看结果..
if result is None or result.rowcount < 1: # or len(result) < 1:
pass
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
# 若没找到，则insert一条到最新的库
# res==True表示至少有一个库里面有一条符合条件的任务，且已更新其字段
if not res:
conn = self.connect_write(5)
try:
# ClientId
# CmdId,
# Platform,
# TaskId,
# BatchId,
# Cmd,
# Status,
# Source,
# CmdRcvMsg,
# CreateTime,
# UpdateTime,
# Sequence
# IsRelative
# insert
cmd = f'''INSERT INTO {self._tbname}(
ClientId,
CmdId,
Platform,
TaskId,
BatchId,
Cmd,
Source,
Status,
CreateTime,
UpdateTime,
Sequence,
IsRelative) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)'''
# 有几个属于TaskBack的字段在此不更新
cursor = conn.cursor
result = cursor.execute(cmd, (
client._statusbasic._clientid,
command.cmd_id,
platform,
taskid,
batchid,
command.cmd_str,
command.source,
cmdstatus.value,
helper_time.timespan_to_datestr_tz(cmdtime),
helper_time.get_time_sec_tz(),
0,
isrelative,
))
if result is None or result.rowcount < 1: # or len(result) < 1:
res = False
else:
res = True
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
except Exception:
self._logger.error("save_new_idowncmd error: %s" %
traceback.format_exc())
return res
@table_locker(__tb_IDownCmd._tbname)
def select_cmd(self, conds: SqlConditions) -> IdownCmd:
"""按条件搜索任务，返回数据行转换成的字段字典迭代器"""
conn: SqliteConn = None
cursor = None
conds: SqlConditions = conds
try:
cmd = f'''SELECT
ClientId,
CmdId,
Platform,
TaskId,
BatchId,
Cmd,
Status,
Source,
CmdRcvMsg,
CreateTime,
UpdateTime,
Sequence,
IsRelative
FROM {self._tbname} WHERE {conds.text_normal}'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
conn._conn.row_factory = self._dict_factory
try:
cursor = conn.cursor
cursor.execute(cmd, conds.params)
result = cursor.fetchall()
if result is None or len(result) < 1:
continue
fields: dict = {}
for i in range(len(result[0])):
fields[cursor.description[i][0].lower()] = result[0][i]
return fields
except Exception:
self._logger.error("Get IDownCmd error: {}".format(
traceback.format_exc()))
finally:
if not conn is None:
conn.close()
except Exception:
self._logger.error("Get IDownCmd error: %s" %
traceback.format_exc())
@table_locker(__tb_IDownCmd._tbname)
def select_cmds(self, conds: SqlConditions) -> iter:
"""按条件搜索任务，返回数据行转换成的字段字典迭代器"""
conn: SqliteConn = None
cursor = None
conds: SqlConditions = conds
try:
cmd = f'''SELECT
ClientId,
CmdId,
Platform,
TaskId,
BatchId,
Cmd,
Status,
Source,
CmdRcvMsg,
CreateTime,
UpdateTime,
Sequence,
IsRelative
FROM {self._tbname} WHERE {conds.text_normal}'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
conn._conn.row_factory = self._dict_factory
try:
cursor = conn.cursor
cursor.execute(cmd, conds.params)
result = cursor.fetchall()
if result is None or len(result) < 1:
continue
for row in result:
# if len(row) != 15:
# continue
fields: dict = {}
for i in range(len(row)):
fields[cursor.description[i][0].lower()] = row[i]
yield fields
except Exception:
self._logger.error("save_idown_task error: {}".format(
traceback.format_exc()))
finally:
if not conn is None:
conn.close()
except Exception:
self._logger.error("Get IDownCmd error: %s" %
traceback.format_exc())
@table_locker(__tb_IDownCmd._tbname)
def update_cmd_status(self, platform: str, cmdid: str, clientid: str,
cmdstatus: ECommandStatus) -> bool:
"""更新cmd的Status状态字段"""
conn: SqliteConn = None
cursor = None
res: bool = False
try:
cmd = f'''UPDATE {self._tbname} set
Status=?
WHERE Platform=? and CmdId=? and ClientId=?;'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
result = cursor.execute(
cmd, (cmdstatus.value, platform, cmdid, clientid))
if not result is None and result.rowcount > 0:
res = True # 一定只有一个子任务
break
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
except Exception:
self._logger.error("update {} Status error: {}".format(
self._tbname, traceback.format_exc()))
return res
@table_locker(__tb_IDownCmd._tbname)
def update_cmd(
self,
cmd: IdownCmd,
taskid: str = None,
batchid: str = None,
isrelative: int = 1,
) -> bool:
"""更新cmd，条件为platform+cmdid，更新其他可更新字段"""
conn: SqliteConn = None
cursor = None
res: bool = False
cmd: IdownCmd = cmd
try:
# ClientId
# CmdId,
# Platform,
# TaskId,
# BatchId,
# Cmd,
# Status,
# Source,
# CmdRcvMsg,
# CreateTime,
# UpdateTime,
# Sequence,
# IsRelative
cmd = f'''UPDATE {self._tbname} set
TaskId=?,
BatchId=?,
Cmd=?,
Status=?,
Source=?,
CmdRcvMsg=?,
CreateTime=?,
UpdateTime=?,
Sequence=?,
IsRelative=?
WHERE Platform=? and CmdId=?;'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
# task:Task = task
result = cursor.execute(
cmd,
(
taskid,
batchid,
cmd.cmd_str,
cmd.cmdstatus.value,
cmd.source,
cmd.cmdrcvmsg,
# 假设数据是按时间先后顺序过来的，所以createtime直接取本地时间。
# 若后面有问题，需要加idowncmd命令文件字段createtime字段
helper_time.get_time_sec_tz(),
helper_time.get_time_sec_tz(),
cmd._sequence,
isrelative,
cmd._platform,
cmd.cmd_id,
))
if not result is None and result.rowcount > 0:
res = True
break
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
except Exception:
self._logger.error("update {} error: {}".format(
self._tbname, traceback.format_exc()))
return res
@table_locker(__tb_IDownCmd._tbname)
def update_cmdback(self, cmdback: CmdFeedBack) -> bool:
"""更新cmdback到数据库，条件为clientid+cmdid(+platform?)，更新其他可更新字段"""
conn: SqliteConn = None
cursor = None
res: bool = False
cmdback: CmdFeedBack = cmdback
try:
if cmdback is None or cmdback._platform is None:
raise Exception("Invalid cmdback obj")
# ClientId
# CmdId,
# Platform,
# TaskId,
# BatchId,
# Cmd,
# Status,
# Source,
# CmdRcvMsg,
# CreateTime,
# UpdateTime,
# Sequence,
# IsRelative
cmd = f'''UPDATE {self._tbname} set
Status=?,
CmdRcvMsg=?,
UpdateTime=?,
Sequence=?
WHERE ClientId=? and CmdId=?;'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
# task:Task = task
result = cursor.execute(cmd, (
cmdback._cmdstatus.value,
cmdback._cmdrcvmsg,
helper_time.get_time_sec_tz(),
cmdback._sequence,
cmdback.clientid,
cmdback.cmd_id,
))
if not result is None and result.rowcount > 0:
res = True
break
else:
self._logger.error(
"Cmd not found in db:\ncmdid={}\nclientid={}".
format(cmdback.cmd_id, cmdback.clientid))
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
except Exception:
self._logger.error("update CmdFeedBack to db {} error: {}".format(
self._tbname, traceback.format_exc()))
return res
def save_new_idownbatchtoken(
self,
task: Task,
) -> bool:
"""保存新的批处理任务的令牌资源\n
task:任务对象"""
res = False
conn: SqliteConn = None
cursor = None
task: Task = task
try:
# 搜索每个库，看有没有 TokenId一样的，且时间更新
# 的，一样就更新其他所有字段
cmd = f'''SELECT COUNT(1) FROM {self._tbname} WHERE
TokenId=?'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
cursor.execute(cmd, (task.tokenid, ))
result = cursor.fetchall()
if result[0][0] > 0:
res = True
# 只根据TaskId、ParentTaskId和ClientId关联，
# 不考虑 任务文件产生时间与现有数据库中已存在任务的时间，每次直接覆盖，以符合用户操作。
# 若来了TaskId一样的数据，则必然分配给同一个ClientId
cmd = f'''UPDATE {self._tbname} set
TokenType=?,
Input=?,
PreGlobalTelCode=?,
PreAccount=?,
GlobalTelCode=?,
Phone=?,
Account=?,
Password=?,
Url=?,
Host=?,
Cookie=?
WHERE TokenId=? and UpdateTime<=?;'''
result = cursor.execute(cmd, (
task.tokentype.value,
task.input,
task.preglobaltelcode,
task.preaccount,
task.globaltelcode,
task.phone,
task.account,
task.password,
task.url,
task.host,
task.cookie,
task.tokenid,
task.time,
))
# 这句没用，就是调试看看结果..
if result is None or result.rowcount < 1: # or len(result) < 1:
pass
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
# 若没找到，则insert一条到最新的库
# res==True表示至少有一个库里面有一条符合条件的任务，且已更新其字段
if not res:
conn = self.connect_write(5)
try:
# insert
cmd = f'''INSERT INTO {self._tbname}(
TokenId,
TokenType,
Input,
PreGlobalTelCode,
PreAccount,
GlobalTelCode,
Phone,
Account,
Password,
Url,
Host,
Cookie,
UpdateTime,
CreateTime) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)'''
# 有几个属于TaskBack的字段在此不更新
cursor = conn.cursor
result = cursor.execute(cmd, (
task.tokenid,
task.tokentype.value,
task.input,
task.preglobaltelcode,
task.preaccount,
task.globaltelcode,
task.phone,
task.account,
task.password,
task.url,
task.host,
task.cookie,
task.time,
helper_time.ts_since_1970_tz(),
))
if result is None or result.rowcount < 1: # or len(result) < 1:
res = False
else:
res = True
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
except Exception:
self._logger.error("save_new_idownbatchtask error: %s" %
traceback.format_exc())
return res
class TbIDownBatchToken(TbSqliteBase):
"""Task表及相关操作"""
__tb_IDownBatchToken: SqliteTable = SqliteTable(
'IDownBatchToken',
True,
SqliteColumn(colname='Id',
coltype='INTEGER',
nullable=False,
is_primary_key=True,
is_auto_increament=True,
is_unique=True).set_index_new(),
SqliteColumn(colname='TokenId', nullable=False).set_index_new(),
SqliteColumn(colname='TokenType', coltype='INTEGER',
nullable=False).set_index_new(),
SqliteColumn(colname='Input'),
SqliteColumn(colname='PreGlobalTelCode'),
SqliteColumn(colname='PreAccount'),
SqliteColumn(colname='GlobalTelCode'),
SqliteColumn(colname='Phone'),
SqliteColumn(colname='Account'),
SqliteColumn(colname='Password'),
SqliteColumn(colname='Url'),
SqliteColumn(colname='Host'),
SqliteColumn(colname='Cookie'),
SqliteColumn(colname='CreateTime',
coltype='DATETIME',
defaultval=helper_time.ts_since_1970_tz()),
SqliteColumn(colname='UpdateTime', coltype='REAL',
nullable=False).set_index_new(),
)
# 所有列，复制粘贴用...：
# TokenId,
# TokenType,
# Input,
# PreGlobalTelCode,
# PreAccount,
# GlobalTelCode,
# Phone,
# Account,
# Password,
# Url,
# Host,
# Cookie,
# UpdateTime,
# CreateTime
def __init__(self, dbcfg: SqliteConfig):
""""""
TbSqliteBase.__init__(self,
TbIDownBatchToken.__tb_IDownBatchToken._tbname,
dbcfg)
def _append_tables(self):
self._conn_mngr.append_table(TbIDownBatchToken.__tb_IDownBatchToken)
@table_locker(__tb_IDownBatchToken._tbname)
def save_new_idownbatchtoken(
self,
task: Task,
) -> bool:
"""保存新的批处理任务的令牌资源\n
task:任务对象"""
res = False
conn: SqliteConn = None
cursor = None
task: Task = task
try:
# 搜索每个库，看有没有 TokenId一样的，且时间更新
# 的，一样就更新其他所有字段
cmd = f'''SELECT COUNT(1) FROM {self._tbname} WHERE
TokenId=?'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
cursor.execute(cmd, (task.tokenid, ))
result = cursor.fetchall()
if result[0][0] > 0:
res = True
# 只根据TaskId、ParentTaskId和ClientId关联，
# 不考虑 任务文件产生时间与现有数据库中已存在任务的时间，每次直接覆盖，以符合用户操作。
# 若来了TaskId一样的数据，则必然分配给同一个ClientId
cmd = f'''UPDATE {self._tbname} set
TokenType=?,
Input=?,
PreGlobalTelCode=?,
PreAccount=?,
GlobalTelCode=?,
Phone=?,
Account=?,
Password=?,
Url=?,
Host=?,
Cookie=?
WHERE TokenId=? and UpdateTime<=?;'''
result = cursor.execute(cmd, (
task.tokentype.value,
task.input,
task.preglobaltelcode,
task.preaccount,
task.globaltelcode,
task.phone,
task.account,
task.password,
task.url,
task.host,
task.cookie,
task.tokenid,
task.time,
))
# 这句没用，就是调试看看结果..
if result is None or result.rowcount < 1: # or len(result) < 1:
pass
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
# 若没找到，则insert一条到最新的库
# res==True表示至少有一个库里面有一条符合条件的任务，且已更新其字段
if not res:
conn = self.connect_write(5)
try:
# insert
cmd = f'''INSERT INTO {self._tbname}(
TokenId,
TokenType,
Input,
PreGlobalTelCode,
PreAccount,
GlobalTelCode,
Phone,
Account,
Password,
Url,
Host,
Cookie,
UpdateTime,
CreateTime) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)'''
# 有几个属于TaskBack的字段在此不更新
cursor = conn.cursor
result = cursor.execute(cmd, (
task.tokenid,
task.tokentype.value,
task.input,
task.preglobaltelcode,
task.preaccount,
task.globaltelcode,
task.phone,
task.account,
task.password,
task.url,
task.host,
task.cookie,
task.time,
helper_time.ts_since_1970_tz(),
))
if result is None or result.rowcount < 1: # or len(result) < 1:
res = False
else:
res = True
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
except Exception:
self._logger.error("save_new_idownbatchtask error: %s" %
traceback.format_exc())
return res
@table_locker(__tb_IDownBatchToken._tbname)
def select_token(self, conds: SqlConditions) -> iter:
"""按条件搜索任务，返回数据行转换成的字段字典迭代器"""
conn: SqliteConn = None
cursor = None
conds: SqlConditions = conds
try:
cmd = f'''SELECT
TokenId,
TokenType,
Input,
PreGlobalTelCode,
PreAccount,
GlobalTelCode,
Phone,
Account,
Password,
Url,
Host,
Cookie,
UpdateTime,
CreateTime FROM {self._tbname} WHERE {conds.text_normal}'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
conn._conn.row_factory = self._dict_factory
try:
cursor = conn.cursor
cursor.execute(cmd, conds.params)
result = cursor.fetchall()
if result is None or len(result) < 1:
continue
for row in result:
# if len(row) != 15:
# continue
fields: dict = {}
for i in range(len(row)):
fields[cursor.description[i][0].lower()] = row[i]
yield fields
except Exception:
self._logger.error("save_idown_task error: {}".format(
traceback.format_exc()))
finally:
if not conn is None:
conn.close()
except Exception:
self._logger.error("Save new idown_task error: %s" %
traceback.format_exc())
@table_locker(__tb_IDownBatchToken._tbname)
def select_token_one(self, conds: SqlConditions) -> dict:
"""按条件搜索任务，返回数据行转换成的字段字典"""
for fields in self.select_token(conds):
return fields
def _update_to_db(self, task: IscoutTask, targetclient: Client,
succ: bool):
"""同步任务下发状态到数据库。这里来的永远是子任务"""
try:
task.laststarttime = helper_time.ts_since_1970_tz()
updatefields: dict = {}
updatefields["PeriodNum"] = task.periodnum
if not succ:
# 失败，返回回馈数据
task.lastendtime = helper_time.ts_since_1970_tz()
updatefields["LastEndTime"] = task.lastendtime
task.cmdstatus = ECommandStatus.Failed
task.cmdrcvmsg = "发送任务到采集端失败"
tb: IscoutBtaskBack = IscoutBtaskBack.create_from_task(
task, cmdstatus=task.cmdstatus, recvmsg=task.cmdrcvmsg)
if not OutputManagement.output(tb):
self._logger.error(
"Write IscoutBtaskBack failed:\ntaskid:{}".format(
task.taskid))
if isinstance(task.cmd, IdownCmd):
task.cmd.cmdstatus = ECommandStatus.Failed
task.cmd.cmdrcvmsg = task.cmdrcvmsg
cmdback: CmdFeedBack = CmdFeedBack.create_from_cmd(
task.cmd, task.cmd.cmdstatus, task.cmd.cmdrcvmsg)
if not OutputManagement.output(cmdback):
self._logger.error(
"Write IDownCmdBack failed:\nplatform:{}\ncmdid:{}"
.format(task._platform, task.cmd_id))
else:
# 成功
task.cmdstatus = ECommandStatus.Dealing
if isinstance(task.cmd, IdownCmd):
task.cmd.cmdstatus = ECommandStatus.Dealing
# 返回 子任务开始回馈
task.cmdstatus = ECommandStatus.Dealing
task.cmdrcvmsg = "任务开始执行"
if task.periodnum > 1:
task.cmdrcvmsg += ",周期{}".format(task.periodnum)
tb: IscoutBtaskBack = IscoutBtaskBack.create_from_task(
task, cmdstatus=task.cmdstatus, recvmsg=task.cmdrcvmsg)
if not OutputManagement.output(tb):
self._logger.error(
"Write IscoutBtaskBack failed:\ntaskid:{}".format(
task.taskid))
self._logger.info(
"Send IScoutTask succeed [{}: {}]:\ntaskid:{}\nbatchid:{}\nperiod:{}\nclient:{}\t{}"
.format(
task._objecttype.name,
task._object,
task.taskid,
task.batchid,
task.periodnum,
task._clientid,
targetclient._statusbasic.ip,
))
with self._dbmanager.get_automic_locker(
EDBAutomic.ReduplicateIScoutTask):
# 总任务batchcompletecount -1
# 子任务isbatchcompletecountincreaced 置0
self._dbmanager.decreace_iscouttask_batch_complete_count(
task._platform, task.taskid, task.batchid)
task.batchcompletecount -= 1
# 更新数据库当前子任务状态
updatefields["Status"] = task.cmdstatus.value
updatefields["LastStartTime"] = task.laststarttime
if not self._dbmanager.update_iscoutbtask(
task._platform, task.taskid, task.batchid,
updatefields):
# 这里如果更新数据库状态失败了，那发送线程可能会重新发送当前子任务。
# 此时需要采集端支持 冥等性（任务去重，保证多次收到同一条任务只会处理一条）
self._logger.error(
"Update IScoutBatchTask cmdstatus failed:\ntaskid:{}\ncmdstatus:{}"
.format(task.taskid, task.cmdstatus.name))
# 更新总任务开始时间
if not self._dbmanager.update_iscout_task(
task._platform,
task.taskid,
updatefields={
"LastStartTime": helper_time.ts_since_1970_tz(),
"PeriodNum": task.periodnum,
},
):
self._logger.error(
"Update IScouttask LastStartTime failed:\ntaskid:{}\ncmdstatus:{}"
.format(task.taskid, task.cmdstatus.name))
# 如果带cmd，更新cmd状态
if isinstance(
task.cmd,
IdownCmd) and not self._dbmanager.update_cmd_status(
task.cmd.platform,
task.cmd.cmd_id,
targetclient._statusbasic._clientid
if not targetclient is None else task._clientid,
task.cmd.cmdstatus,
):
self._logger.error(
"Update cmd cmdstatus failed:\ntaskid:{}\ncmdstatus:{}"
.format(task.taskid, task.cmdstatus.name))
# 返回 总任务开始回馈
task.cmdstatus = ECommandStatus.Dealing
task.cmdrcvmsg = "任务开始执行，周期{}".format(task.periodnum)
tb: IscoutTaskBack = IscoutTaskBack.create_from_task(
task,
cmdstatus=task.cmdstatus,
recvmsg=task.cmdrcvmsg,
batchcompletecount=task.batchcompletecount,
periodnum=task.periodnum,
)
if not OutputManagement.output(tb):
self._logger.error(
"Write IscoutBtaskBack failed:\ntaskid:{}".format(
task.taskid))
except Exception:
self._logger.error("Update task cmdstatus to db error: {}".format(
traceback.format_exc()))
self._dbmanager.update_iscouttask_status(task._platform,
task.taskid,
ECommandStatus.Failed)
if isinstance(task.cmd, IdownCmd):
self._dbmanager.update_cmd_status(
task._platform,
task.cmd_id,
task.cmd._clientid,
ECommandStatus.Failed,
)
def _generate_task_by_config(self, autotaskconfig: AutoTaskConfig) -> iter:
"""generate出来是多个子任务"""
lastendtime: str = None
periodnum: int = None
# 从库中找
# 1. 查询最晚的一条总任务
# 2. 条件1：若有，则判断其完成时间是否超过周期循环间隔时间 或
# 3. 条件2：若无
# 4. 则下发一条全新的、周期数+1的任务
# 这样才能记录每一次任务的每个子任务被分发到的ClientId
got: bool = False
task: AutomatedTask = self._dbmanager.get_last_automatedtask(
autotaskconfig._platform, autotaskconfig._tasktype)
if isinstance(task, AutomatedTask):
got = True
lastendtime = task.lastendtime
periodnum = task.periodnum
do: bool = False
if got and lastendtime is None:
# 已经有任务，但是还么执行完过
return
elif not got:
# 新部署的，还没有过任务
periodnum = 0
do = True
else:
# got and lastendtime is not None的情况
# tend = time.mktime(
# time.strptime(task.lastendtime, "%Y-%m-%d %H:%M:%S"))
tend = task.lastendtime
tstart = tend + task.cmd.stratagy.interval
tnow = helper_time.ts_since_1970_tz()
if tnow > tstart:
do = True
if not do:
return
# 造任务
taskid: str = str(uuid.uuid1())
count: int = 1
if autotaskconfig._seperatetask:
count = len(StatusMantainer.get_clientstatus_id_sorted())
for i in range(count):
fields: dict = {
"platform":
autotaskconfig._platform,
"source":
autotaskconfig._source,
"periodnum":
periodnum + 1,
"taskid":
taskid,
"batchid":
str(uuid.uuid1()),
"autotasktype":
autotaskconfig._tasktype.value,
"createtime":
helper_time.ts_since_1970_tz(),
"cmdid":
str(uuid.uuid1()),
"cmd":
json.dumps({
"stratagy": {
"circulation_mode": 2,
"interval": autotaskconfig._interval,
},
"stratagyauto": {
"section": "1/{}".format(count),
"index": i
}
}),
}
task: AutomatedTask = AutomatedTask.create_from_dict(
fields, autotaskconfig._platform)
yield task
class TbIDownBatchTask(TbSqliteBase):
"""Task表及相关操作"""
__tb_IDownBatchTask: SqliteTable = SqliteTable(
'IDownBatchTask',
True,
SqliteColumn(colname='Id',
coltype='INTEGER',
nullable=False,
is_primary_key=True,
is_auto_increament=True,
is_unique=True).set_index_new(),
SqliteColumn(colname='Platform', nullable=False).set_index_new(),
SqliteColumn(colname='TaskId', nullable=False).set_index_new(),
SqliteColumn(colname='BatchId', nullable=False).set_index_new(),
SqliteColumn(colname='ParentBatchId', nullable=True).set_index_new(),
SqliteColumn(colname='ClientId', nullable=False).set_index_new(),
SqliteColumn(colname='TokenId', coltype='INTEGER',
nullable=False).set_index_new(),
SqliteColumn(colname='AppType', coltype='INTEGER').set_index_new(),
SqliteColumn(colname='ForceDownload', coltype='INTEGER',
defaultval=0).set_index_new(),
SqliteColumn(colname='CmdRcvMsg'),
SqliteColumn(colname='Result'),
SqliteColumn(colname='CmdStatus', coltype='INTEGER',
nullable=False).set_index_new(),
SqliteColumn(
colname='Progress', #0~1浮点数表百分比
coltype='REAL',
defaultval=0).set_index_new(),
SqliteColumn(colname='Sequence', coltype='INTEGER',
defaultval=0).set_index_new(),
SqliteColumn(colname='OtherFields').set_index_new(),
SqliteColumn(colname='CreateTime',
coltype='DATETIME',
defaultval=helper_time.ts_since_1970_tz()),
SqliteColumn(colname='UpdateTime', coltype='REAL',
nullable=False).set_index_new(),
SqliteColumn(
# 此字段仅在查询 或 操作IDownTask表的 BatchCompleteCount 字段时使用，
# 其他时候不要使用
colname='IsBatchCompleteCountIncreased',
coltype='BOOLEAN',
nullable=False,
defaultval=0),
SqliteColumn(colname="Source"), # 新增
SqliteColumn(colname="CmdId"), #cmdid字段要更新
)
# 所有列，复制粘贴用...：
# Platform,
# TaskId,
# Batchid,
# ParentBatchId,
# ClientId,
# TokenId,
# AppType,
# ForceDownload,
# CmdRcvMsg,
# Result,
# CmdStatus,
# Progress,
# Sequence,
# OtherFields,
# CreateTime,
# UpdateTime,
# IsBatchCompleteCountIncreased,
# Source,
# CmdId
def __init__(self, dbcfg: SqliteConfig):
""""""
TbSqliteBase.__init__(self,
TbIDownBatchTask.__tb_IDownBatchTask._tbname,
dbcfg)
def _append_tables(self):
self._conn_mngr.append_table(TbIDownBatchTask.__tb_IDownBatchTask)
@table_locker(__tb_IDownBatchTask._tbname)
def save_new_idownbatchtask(
self,
task: Task,
client: Client,
cmdstatus: ECommandStatus = ECommandStatus.WaitForSend
) -> Tuple[bool, bool]:
"""保存新的批处理任务的子任务\n
task: 任务对象\n
tokenid：当前批处理子任务对应的登陆令牌存储在IDownBatchTokens表中的tokenid，关联存储\n
client: 当前批处理子任务被分配到的client\n
cmdstatus: 指定当前批处理子任务的命令状态\n
return: （bool是否成功，bool是否为新增(用于计算batchtotalcount)）"""
res: bool = False
isnew: bool = False
conn: SqliteConn = None
cursor = None
client: Client = client
task: Task = task
try:
if not isinstance(task.tokenid, str) or task.tokenid == "":
raise Exception(
"Invalid tokenid while save new idownbatchtask:\ntaskid:{}"
.format(task.taskid))
# 搜索每个库，看有没有 taskid和clientid一样，且时间更新
# 的，一样就更新其他所有字段
cmd = f'''SELECT COUNT(1) FROM {self._tbname} WHERE
Platform=? and TaskId=? and BatchId=? and ClientId=?'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
cursor.execute(cmd, (
task._platform,
task.taskid,
task.batchid,
client._statusbasic._clientid,
))
result = cursor.fetchall()
if result[0][0] > 0:
res = True
# 只根据TaskId、ParentTaskId和ClientId关联，
# 不考虑 任务文件产生时间与现有数据库中已存在任务的时间，每次直接覆盖，以符合用户操作。
# 若来了TaskId一样的数据，则必然分配给同一个ClientId
cmd = f'''UPDATE {self._tbname} set
TokenId=?,
AppType=?,
ForceDownload=?,
CmdRcvMsg=?,
Result=?,
CmdStatus=?,
Progress=?,
Sequence=?,
OtherFields=?,
CreateTime=?,
UpdateTime=?,
Source=?,
CmdId=?
WHERE Platform=? and TaskId=? and BatchId=? and ClientId=? and CreateTime<=?;'''
result = cursor.execute(
cmd,
(
task.tokenid,
task.apptype,
task.forcedownload,
task._cmdrcvmsg,
task._result,
cmdstatus.value,
# 进度置零
0,
#批量任务完成数也要重置，因为重置了任务下发状态，变为新任务，需要重新执行
0, #sequence重置
task.other_fields_json,
task.timestr,
helper_time.get_time_sec_tz(),
task.source,
task.cmd_id,
# 0 此字段在其他地方重置 #是否总任务已提升batchcompletecount置零，
task._platform,
task.taskid,
task.batchid,
client._statusbasic._clientid,
task.timestr,
))
# 这句没用，就是调试看看结果..
if result is None or result.rowcount < 1: # or len(result) < 1:
pass
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
# 若没找到，则insert一条到最新的库
# res==True表示至少有一个库里面有一条符合条件的任务，且已更新其字段
if not res:
isnew = True
conn = self.connect_write(5)
try:
# insert
# Platform,
# TaskId,
# Batchid,
# ParentBatchId,
# ClientId,
# TokenId,
# AppType,
# ForceDownload,
# CmdRcvMsg,
# Result,
# CmdStatus,
# Progress,
# Sequence,
# OtherFields,
# CreateTime,
# UpdateTime
# IsBatchCompleteCountIncreased
cmd = f'''INSERT INTO {self._tbname}(
Platform,
TaskId,
Batchid,
ParentBatchId,
ClientId,
TokenId,
AppType,
ForceDownload,
CmdRcvMsg,
Result,
CmdStatus,
Progress,
Sequence,
OtherFields,
CreateTime,
UpdateTime,
Source,
CmdId) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'''
# 有几个属于TaskBack的字段在此不更新
cursor = conn.cursor
result = cursor.execute(
cmd,
(
task._platform,
task.taskid,
task.batchid,
task.parentbatchid,
client._statusbasic._clientid,
task.tokenid,
task.apptype,
task.forcedownload,
task._cmdrcvmsg,
task._result,
cmdstatus.value,
0, # progress初始化为0
0, #sequence初始化为0
task.other_fields_json,
task.timestr,
helper_time.get_time_sec_tz(),
task.source,
task.cmd_id))
if result is None or result.rowcount < 1: # or len(result) < 1:
res = False
else:
res = True
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
except Exception:
self._logger.error("save_new_idownbatchtask error: %s" %
traceback.format_exc())
return (res, isnew)
@table_locker(__tb_IDownBatchTask._tbname)
def get_parent_clientid_of_task(self, task: Task) -> str:
"""获取 指定task的被分配到的采集端 Client对象"""
res: str = None
conn: SqliteConn = None
cursor = None
task: Task = task
try:
if helper_str.is_none_or_empty(task.parenttaskid):
raise Exception(
"Invalid task parent_taskid for task, taskid={} batchid={}"
.format(task.taskid, task.batchid))
if helper_str.is_none_or_empty(task.parentbatchid):
raise Exception(
"Invalid task parent_batchid for task, taskid={} batchid={}"
.format(task.taskid, task.batchid))
# 搜索每个库，看有没有 taskid和clientid一样，且时间更新
# 的，一样就更新其他所有字段
cmd = f'''SELECT ClientId FROM {self._tbname} WHERE
Platform=? and TaskId=? and BatchId=?'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
cursor.execute(cmd, (
task._platform,
task.parenttaskid,
task.parentbatchid,
))
result = cursor.fetchall()
if result is None or len(result) < 1 or len(result[0]) < 1:
continue
else:
res = result[0][0]
break
except Exception:
self._logger.error(
"get_parent_client_of_task error: {}".format(
traceback.format_exc()))
finally:
if not conn is None:
conn.close()
if not res is None:
break
except Exception:
self._logger.error("get_parent_client_of_task error: %s" %
traceback.format_exc())
return res
@table_locker(__tb_IDownBatchTask._tbname)
def get_batch_task(self, conds: SqlConditions) -> dict:
"""按条件搜索指定的一个任务，返回数据行转换成的字段字典迭代器"""
conn: SqliteConn = None
cursor = None
conds: SqlConditions = conds
try:
cmd = f'''SELECT
Platform,
TaskId,
Batchid,
ParentBatchId,
ClientId,
TokenId,
AppType,
ForceDownload,
CmdRcvMsg,
Result,
CmdStatus,
Progress,
Sequence,
OtherFields,
CreateTime,
UpdateTime,
IsBatchCompleteCountIncreased,
Source,
CmdId FROM {self._tbname} WHERE {conds.text_normal}'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
conn._conn.row_factory = self._dict_factory
# conn._conn.text_factory = self._text_factory
try:
cursor = conn.cursor
cursor = cursor.execute(cmd, conds.params)
result = cursor.fetchall()
if result is None or len(result) < 1:
continue
fields: dict = {}
for i in range(len(result[0])):
fields[cursor.description[i][0].lower()] = result[0][i]
return fields
except Exception:
self._logger.error(
"Get BatchTask from {} error: {}".format(
self._tbname, traceback.format_exc()))
finally:
if not conn is None:
conn.close()
except Exception:
self._logger.error("Get BatchTask {} error: {}".format(
self._tbname, traceback.format_exc()))
@table_locker(__tb_IDownBatchTask._tbname)
def get_batch_tasks(self, conds: SqlConditions) -> iter:
"""按条件搜索任务，返回数据行转换成的字段字典迭代器"""
conn: SqliteConn = None
cursor = None
conds: SqlConditions = conds
try:
cmd = f'''SELECT
Platform,
TaskId,
Batchid,
ParentBatchId,
ClientId,
TokenId,
AppType,
ForceDownload,
CmdRcvMsg,
Result,
CmdStatus,
Progress,
Sequence,
OtherFields,
CreateTime,
UpdateTime,
IsBatchCompleteCountIncreased,
Source,
CmdId FROM {self._tbname} WHERE {conds.text_normal}'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
conn._conn.row_factory = self._dict_factory
# conn._conn.text_factory = self._text_factory
try:
cursor = conn.cursor
cursor = cursor.execute(cmd, conds.params)
result = cursor.fetchall()
if result is None or len(result) < 1:
continue
for row in result:
# if len(row) != 15:
# continue
fields: dict = {}
for i in range(len(row)):
fields[cursor.description[i][0].lower()] = row[i]
yield fields
except Exception:
self._logger.error(
"Get BatchTasks from {} error: {}".format(
self._tbname, traceback.format_exc()))
finally:
if not conn is None:
conn.close()
except Exception:
self._logger.error("Get BatchTasks {} error: {}".format(
self._tbname, traceback.format_exc()))
@table_locker(__tb_IDownBatchTask._tbname)
def get_batch_task_count_by_cmdstatus(self, task: Task,
cmdstatus: ECommandStatus) -> int:
"""查询指定总任务taskid下的所有 为指定命令状态的子任务 的数量"""
res: int = 0 #总数量
conn: SqliteConn = None
cursor = None
try:
cmd = f'''SELECT COUNT() FROM {self._tbname} WHERE Platform=? and TaskId=? and CmdStatus=?;'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
cursor.execute(cmd, (
task._platform,
task.taskid,
cmdstatus.value,
))
result = cursor.fetchall()
if result is None or len(result) < 1:
continue
tmp = result[0][0]
res = res + tmp
except Exception:
self._logger.error("Select {} error: {}".format(
self._tbname, traceback.format_exc()))
finally:
if not conn is None:
conn.close()
except Exception:
self._logger.error("Save new {} error: {}".format(
self._tbname, traceback.format_exc()))
return res
@table_locker(__tb_IDownBatchTask._tbname)
def update_batchtask(self, task: Task) -> bool:
"""使用指定task对象更新IDownBatchTask表，
不更新 isbatchcompletecountincreased\n
task: 表示一个子任务对象"""
conn: SqliteConn = None
cursor = None
res: bool = False
try:
task._cmdstatus
cmd = f'''UPDATE {self._tbname} set
ClientId=?,
ParentBatchId=?,
TokenId=?,
AppType=?,
ForceDownload=?,
CmdRcvMsg=?,
Result=?,
CmdStatus=?,
Progress=?,
Sequence=?,
OtherFields=?,
CreateTime=?,
UpdateTime=?,
Source=?,
CmdId=?
WHERE Platform=? and TaskId=? and BatchId=?;'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
# task:Task = task
result = cursor.execute(cmd, (
task._clientid,
task.parentbatchid,
task.tokenid,
task.apptype,
task.forcedownload,
task._cmdrcvmsg,
task._result,
task._cmdstatus.value,
task.progress,
task._sequence,
task.other_fields_json,
task.timestr,
helper_time.get_time_sec_tz(),
task.source,
task.cmd_id,
task._platform,
task.taskid,
task.batchid,
))
if not result is None and result.rowcount > 0:
res = True # 一定只有一个子任务
break
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
except Exception:
self._logger.error("update {} error: {}".format(
self._tbname, traceback.format_exc()))
return res
@table_locker(__tb_IDownBatchTask._tbname)
def update_batchtask_status(self, platform: str, taskid: str, batchid: str,
cmdstatus: ECommandStatus) -> bool:
"""使用指定task对象更新IDownBatchTask表，
不更新 isbatchcompletecountincreased\n
task: 表示一个子任务对象"""
conn: SqliteConn = None
cursor = None
res: bool = False
try:
cmd = f'''UPDATE {self._tbname} set
CmdStatus=?
WHERE Platform=? and TaskId=? and BatchId=?;'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
# task:Task = task
result = cursor.execute(cmd, (
cmdstatus.value,
platform,
taskid,
batchid,
))
if not result is None and result.rowcount > 0:
res = True # 一定只有一个子任务
break
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
except Exception:
self._logger.error("update {} error: {}".format(
self._tbname, traceback.format_exc()))
return res
@table_locker(__tb_IDownBatchTask._tbname)
def update_batchtask_client(self, platform: str, taskid: str, batchid: str,
clientid: str) -> bool:
"""更新batchtask被分配到的采集端"""
conn: SqliteConn = None
cursor = None
res: bool = False
try:
if not isinstance(clientid, str) or clientid == "":
raise Exception(
"Invalid clientid for update_batchtask_clientid")
cmd = f'''UPDATE {self._tbname} set
ClientId=?
WHERE Platform=? and TaskId=? and BatchId=?;'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
# task:Task = task
result = cursor.execute(cmd, (
clientid,
platform,
taskid,
batchid,
))
if not result is None and result.rowcount > 0:
res = True # 一定只有一个子任务
break
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
except Exception:
self._logger.error(
"update batchtask client to {} error: {}".format(
self._tbname, traceback.format_exc()))
return res
@table_locker(__tb_IDownBatchTask._tbname)
def update_batchtask_back(self, tb: TaskBatchBack) -> bool:
"""使用指定 TaskBatchBack 对象更新IDownBatchTask表，
不更新 isbatchcompletecountincreased\n"""
conn: SqliteConn = None
cursor = None
res: bool = False
tb: TaskBatchBack = tb
try:
if not isinstance(tb, TaskBatchBack):
self._logger.error(
"Invalid param TaskBatchBack: {}".format(tb))
return res
# Platform
# TaskId,
# Batchid,
# ClientId,
# TokenId,
# AppType,
# CmdRcvMsg,
# Result,
# CmdStatus,
# Progress,
# Sequence,
# OtherFields,
# CreateTime,
# UpdateTime,
# IsBatchCompleteCountIncreased
# CmdId
# 更新策略，先搜一下有没有，并把sequence搜出来，如果
# 本地sequence和新来的任务的sequence一样，则说明
# 采集端的sequence出错了，打一句日志并返回False
cmd = f'''SELECT Sequence FROM {self._tbname} WHERE Platform=? and TaskId=? and BatchId=?;'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
cursor.execute(cmd, (
tb._platform,
tb._taskid,
tb._batchid,
))
result = cursor.fetchall()
if result is None or len(result) < 1:
continue
oldseq = result[0][0]
if oldseq >= tb._sequence:
self._logger.error(
"The comming TaskBatchBack.sequence is {}, but which in local db is {}:\ntaskid:{}\nbatchid:{}\nsequence:{}"
.format(tb._sequence, oldseq, tb._taskid,
tb._batchid, oldseq))
break
cmd = f'''UPDATE {self._tbname} set
CmdRcvMsg=?,
Result=?,
CmdStatus=?,
Progress=?,
Sequence=?,
UpdateTime=?
WHERE Platform=? and TaskId=? and BatchId=? and Sequence<?;'''
cursor = conn.cursor
result = cursor.execute(cmd, (
tb._cmdrcvmsg,
tb._result,
tb._cmdstatus.value,
tb._progress,
tb._sequence,
helper_time.get_time_sec_tz(),
tb._platform,
tb._taskid,
tb._batchid,
tb._sequence,
))
if not result is None and result.rowcount > 0:
res = True # 一定只有一个子任务
break
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
except Exception:
self._logger.error("update {} error: {}".format(
self._tbname, traceback.format_exc()))
return res
@table_locker(__tb_IDownBatchTask._tbname)
def is_batch_complete_count_increaced(self, platform: str, taskid: str,
batchid: str) -> bool:
"""返回 指定的子任务完成情况 是否已更新到 总任务表的 batchcompletecount 字段"""
conn: SqliteConn = None
cursor = None
res: bool = False
try:
# IsBatchCompleteCountIncreased
cmd = f'''SELECT IsBatchCompleteCountIncreased FROM {self._tbname}
WHERE Platform=? and TaskId=? and BatchId=?'''
found = False
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
cursor.execute(cmd, (
platform,
taskid,
batchid,
))
result = cursor.fetchall()
if not result is None and len(result) > 0:
tmp = result[0][0]
res = bool(tmp) # 一定只有一个子任务
found = True
break
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if found:
break
except Exception:
self._logger.error("update {} error: {}".format(
self._tbname, traceback.format_exc()))
return res
@table_locker(__tb_IDownBatchTask._tbname)
def update_batch_complete_count_increaced_flag(
self, platform: str, taskid: str, batchid: str,
isbatchcompletecountincreased: bool) -> bool:
"""使用指定 TaskBatchBack 对象的 isbatchcompletecountincreased 字段\n"""
conn: SqliteConn = None
cursor = None
res: bool = False
try:
# Platform
# TaskId,
# Batchid,
# ClientId,
# TokenId,
# AppType,
# CmdRcvMsg,
# Result,
# CmdStatus,
# Sequence,
# OtherFields,
# CreateTime,
# UpdateTime,
# IsBatchCompleteCountIncreased
cmd = f'''UPDATE {self._tbname} set
IsBatchCompleteCountIncreased=?
WHERE Platform=? and TaskId=? and BatchId=?;'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
result = cursor.execute(cmd, (
1 if isbatchcompletecountincreased else 0,
platform,
taskid,
batchid,
))
if not result is None and result.rowcount > 0:
res = True # 一定只有一个子任务
break
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
except Exception:
self._logger.error("update {} error: {}".format(
self._tbname, traceback.format_exc()))
return res
def _taskback_deal(self):
"""总任务回馈数据处理线程，失败重试机制"""
got: bool = False
while True:
try:
got = False
task: AutomatedTask = self._tbqueue.get(timeout=3)
got = True
# 返回总任务回馈
# 百分比回馈数据若失败重试，会涉及到sequence不对问题，暂时不重试
# if task.batchtotalcount > task.batchcompletecount:
# 如果总任务下的 部分子任务 是已完成状态，
# 则返回总任务回馈数据百分比
# 不是每一个子任务回馈数据都要对应返回一个总任务完成百分比
# 可以根据一定条件返回，比如取模
seq_in_db: int = self._dbmanager.get_autotask_sequence(
task._platform, task.taskid)
task.sequence_set(seq_in_db)
self._generate_task_back_percent(task)
# 返回一次总任务已完成，由于sequence问题暂时不做失败重试
if task.batchtotalcount <= task.batchcompletecount:
# 判定总任务是否完成，并返回总任务回馈数据
# 只要有一个成功的，就当作成功的返回
succcount: int = self._dbmanager.get_autobtask_count_by_cmdstatus(
task._platform, task.taskid, ECommandStatus.Succeed)
if succcount > 0:
task.cmdstatus = ECommandStatus.Succeed
task.cmdrcvmsg = "任务执行完成"
else:
task.cmdstatus = ECommandStatus.Failed
task.cmdrcvmsg = "任务执行失败"
# 先回传 AutomatedTaskBack 回中心，提升其 sequence 字段计数
# 若回传失败，则再上面加的那个循环检查线程中重复回传此回馈数据
taskback: AutotaskBack = AutotaskBack.create_from_task(
task,
task.cmdstatus,
task.cmdrcvmsg,
batchcompletecount=task.batchtotalcount)
if not OutputManagement.output(taskback):
self._logger.error(
"Output AutomatedTaskBack failed:\ntaskid={}".
format(task.taskid))
else:
self._logger.info(
"AutomatedTaskBack generated [{}]:\ntaskid={}".
format(task.cmdstatus.name, task.taskid))
# 更新总任务状态到数据库
# （加一个线程检测是否有总任务一直没有更新状态的，检查是否应该更新，并执行更新操作）
# task.periodnum += 1 子任务那加了1的
task.lastendtime = helper_time.ts_since_1970_tz()
with self._dbmanager.get_automic_locker(
EDBAutomic.ReduplicateAutoTask):
if not self._dbmanager.update_automated_task2(task):
self._logger.error(
"Update AutomatedTask status failed:\ntaskid={}\ncmdstatus={}"
.format(task.taskid, task.cmdstatus.name))
except queue.Empty:
pass
except Exception:
self._logger.error("Taskback deal error: {}".format(
traceback.format_exc()))
finally:
if got:
self._tbqueue.task_done()
def update_automatedtask2(self, task: AutomatedTask) -> bool:
"""更新AutomatedTask表，根据platform,taskid更新其他所有字段"""
res = False
conn: SqliteConn = None
cursor = None
task: AutomatedTask = task
try:
cmd = f'''SELECT COUNT(1) FROM {self._tbname} WHERE Platform=? and TaskId=?'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
cursor.execute(cmd, (task._platform, task.taskid))
result = cursor.fetchall()
if result[0][0] > 0:
# 只根据TaskId、platform作为条件，
# 不考虑 任务文件产生时间与现有数据库中已存在任务的时间，每次直接覆盖，以符合用户操作。
# 若来了TaskId一样的数据，则必然分配给同一个ClientId
cmd = f'''UPDATE {self._tbname} set
IsPeriod=?,
PeriodNum=?,
Interval=?,
LastStartTime=?,
LastEndTime=?,
Status=?,
BatchTotalCount=?,
BatchCompleteCount=?,
Progress=?,
Source=?,
CmdRcvMsg=?,
Sequence=?,
UpdateTime=? WHERE TaskId=? and Platform=?;'''
result = cursor.execute(cmd, (
1 if task._is_period else 0,
task.periodnum,
task.cmd.stratagy.interval,
task.laststarttime,
task.lastendtime,
task.cmdstatus.value,
task.batchtotalcount,
task.batchcompletecount,
task.progress,
task.source,
task.cmdrcvmsg,
task.sequence,
helper_time.ts_since_1970_tz(),
task.taskid,
task.platform,
))
if result is None or result.rowcount < 1: # or len(result) < 1:
continue
else:
res = True
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
except Exception:
self._logger.error(
"Update AutomatedTask error: %s" % traceback.format_exc())
return res
def _update_to_db(self, task: AutomatedTask, targetclient: Client,
succ: bool):
"""同步任务下发状态到数据库。这里来的永远是子任务"""
try:
task.laststarttime = helper_time.ts_since_1970_tz()
updatefields: dict = {}
if not succ:
# 失败，返回回馈数据
task.periodnum += 1
updatefields['PeriodNum'] = task.periodnum
task.lastendtime = helper_time.ts_since_1970_tz()
updatefields['LastEndTime'] = task.lastendtime
task.cmdstatus = ECommandStatus.Failed
task.cmdrcvmsg = "发送任务到采集端失败"
tb: AutotaskBatchBack = AutotaskBatchBack.create_from_task(
task, cmdstatus=task.cmdstatus, recvmsg=task.cmdrcvmsg)
if not OutputManagement.output(tb):
self._logger.error(
"Write IscoutBtaskBack failed:\ntaskid:{}".format(
task.taskid))
# 自动化任务暂不回传cmdback
# if isinstance(task.cmd, IdownCmd):
# task.cmd.cmdstatus = ECommandStatus.Failed
# task.cmd.cmdrcvmsg = task.cmdrcvmsg
# cmdback: CmdFeedBack = CmdFeedBack.create_from_cmd(
# task.cmd, task.cmd.cmdstatus, task.cmd.cmdrcvmsg)
# if not OutputManagement.output(cmdback):
# self._logger.error(
# "Write IDownCmdBack failed:\nplatform:{}\ncmdid:{}"
# .format(task._platform, task.cmd_id))
else:
# 成功
task.cmdstatus = ECommandStatus.Dealing
if isinstance(task.cmd, IdownCmd):
task.cmd.cmdstatus = ECommandStatus.Dealing
# 返回 子任务开始回馈
task.cmdstatus = ECommandStatus.Dealing
task.cmdrcvmsg = "任务开始执行"
if task.periodnum > 1:
task.cmdrcvmsg += ",周期{}".format(task.periodnum)
tb: AutotaskBatchBack = AutotaskBatchBack.create_from_task(
task, cmdstatus=task.cmdstatus, recvmsg=task.cmdrcvmsg)
if not OutputManagement.output(tb):
self._logger.error(
"Write IscoutBtaskBack failed:\ntaskid:{}".format(
task.taskid))
self._logger.info(
"Send AutomatedTask succeed [{}]:\ntaskid:{}\nbatchid:{}\nperiod:{}\nclient:{}\t{}"
.format(task.autotasktype.name, task.taskid, task.batchid,
task.periodnum, task._clientid,
targetclient._statusbasic.ip))
with self._dbmanager.get_automic_locker(
EDBAutomic.ReduplicateAutoTask):
# 更新数据库当前子任务状态
updatefields['Status'] = task.cmdstatus.value
updatefields['LastStartTime'] = task.laststarttime
if not self._dbmanager.update_automated_batch_task(
task._platform, task.taskid, task.batchid,
updatefields):
# 这里如果更新数据库状态失败了，那发送线程可能会重新发送当前子任务。
# 此时需要采集端支持 冥等性（任务去重，保证多次收到同一条任务只会处理一条）
self._logger.error(
"Update AutomatedTask failed:\ntaskid:{}\ncmdstatus:{}"
.format(task.taskid, task.cmdstatus.name))
# 更新总任务开始时间
if not self._dbmanager.update_automated_task(
task._platform,
task.taskid,
updatefields={
'LastStartTime': helper_time.ts_since_1970_tz()
}):
self._logger.error(
"Update IScouttask LastStartTime failed:\ntaskid:{}\ncmdstatus:{}"
.format(task.taskid, task.cmdstatus.name))
# 如果带cmd，更新cmd状态
if isinstance(
task.cmd,
IdownCmd) and not self._dbmanager.update_cmd_status(
task.cmd.platform, task.cmd.cmd_id,
targetclient._statusbasic._clientid,
task.cmd.cmdstatus):
self._logger.error(
"Update cmd cmdstatus failed:\ntaskid:{}\ncmdstatus:{}"
.format(task.taskid, task.cmdstatus.name))
# 返回 总任务开始回馈
task.cmdstatus = ECommandStatus.Dealing
task.cmdrcvmsg = "任务开始执行，周期{}".format(task.periodnum)
tb: AutotaskBack = AutotaskBack.create_from_task(
task,
cmdstatus=task.cmdstatus,
recvmsg=task.cmdrcvmsg,
batchcompletecount=task.batchcompletecount,
periodnum=task.periodnum)
if not OutputManagement.output(tb):
self._logger.error(
"Write IscoutBtaskBack failed:\ntaskid:{}".format(
task.taskid))
except Exception:
self._logger.error("Update task cmdstatus to db error: {}".format(
traceback.format_exc()))
self._dbmanager.update_autotask_status(task._platform, task.taskid,
ECommandStatus.Failed)
if isinstance(task.cmd, IdownCmd):
self._dbmanager.update_cmd_status(task._platform, task.cmd_id,
task.cmd._clientid,
ECommandStatus.Failed)
def get_client_status_all(self, interval: float = 15) -> iter:
"""获取所有采集端任务状态数据。\n
interval: 指定心跳间隔，即只读取最近n秒内更新的采集端状态，单位秒。"""
res: Client = None
conn: SqliteConn = None
cursor = None
try:
# 如果心跳间隔不对，则使用默认值
if not type(interval) in [int, float] or interval < 2:
interval = 15
for conn in self._conn_mngr.connect_all():
# conn = DbSqlite._conn_mngr.connect_write()
conn._conn.row_factory = self._dict_factory
cursor = conn.cursor
cmd = f'''SELECT
ClientId,
SystemVer,
IP,
Mac,
CrossWall,
Country,
Platform,
AppType,
TaskType,
AppClassify,
ClientBusiness,
CpuSize,
CpuPerc,
MemSize,
MemPerc,
BandWidthd,
BandWidthdPerc,
DiskSize,
DiskPerc,
TaskNewCnt,
TaskWaitingCnt,
TaskDownloadingCnt,
UpdateTime
FROM {self._tbname} WHERE UpdateTime>=?'''
# helper_time.ts_since_1970_tz()单位为秒，减去心跳间隔15秒，
# 就是15秒内状态有更新的才认为是上线的，有效的客户端
it = helper_time.ts_since_1970_tz() - interval
cursor.execute(cmd, (it, ))
result = cursor.fetchall()
for row in result:
# if len(row) != 15:
# continue
fields: dict = {}
for i in range(len(row)):
fields[cursor.description[i][0].lower()] = row[i]
sb: StatusBasic = StatusBasic(fields)
st: StatusTask = StatusTask(fields)
res: Client = Client(sb, st)
yield res
except Exception:
self._logger.error(
"Get all client status error: %s" % traceback.format_exc())
finally:
if not conn is None:
conn.close()
def save_new_automatedtask(
self,
task: AutomatedTask,
cmdstatus: ECommandStatus = ECommandStatus.WaitForSend,
) -> bool:
"""保存新的批处理任务的令牌资源\n
task:任务对象"""
res = False
conn: SqliteConn = None
cursor = None
task: AutomatedTask = task
try:
# 搜索每个库，看有没有 TokenId一样的，且时间更新
# 的，一样就更新其他所有字段
cmd = f'''SELECT COUNT(1) FROM {self._tbname} WHERE
TaskId=? and Platform=?'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
cursor.execute(cmd, (
task.taskid,
task._platform,
))
result = cursor.fetchall()
if result[0][0] > 0:
res = True
cmd = f'''UPDATE {self._tbname} set
AutoTaskType=?,
Source=?,
IsPeriod=?,
PeriodNum=?,
Interval=?,
LastStartTime=?,
LastEndTime=?,
Status=?,
CmdRcvMsg=?,
Progress=?,
BatchTotalCount=?,
BatchCompleteCount=?,
Sequence=?
UpdateTime=?,
WHERE TaskId=? and Platform=? and UpdateTime<=?;'''
result = cursor.execute(
cmd,
(
task.autotasktype.value,
task.source,
1 if task._is_period else 0,
task.periodnum,
task.cmd.stratagy.interval,
task.laststarttime,
task.lastendtime,
task.cmdstatus.value,
task.cmdrcvmsg,
task.progress,
task.batchtotalcount,
task.batchcompletecount,
0, #重置sequence
helper_time.ts_since_1970_tz(),
task.taskid,
task._platform,
task.createtime,
))
# 这句没用，就是调试看看结果..
if result is None or result.rowcount > 1: # or len(result) < 1:
pass
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
# 若没找到，则insert一条到最新的库
# res==True表示至少有一个库里面有一条符合条件的任务，且已更新其字段
if not res:
conn = self.connect_write(5)
try:
# insert
cmd = f'''INSERT INTO {self._tbname}(
Platform,
TaskId,
AutoTaskType,
IsPeriod,
PeriodNum,
Interval,
LastStartTime,
LastEndTime,
Status,
BatchTotalCount,
BatchCompleteCount,
Progress,
Source,
CmdRcvMsg,
CreateTime,
UpdateTime,
Sequence) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'''
# 有几个属于TaskBack的字段在此不更新
cursor = conn.cursor
result = cursor.execute(cmd, (
task._platform,
task.taskid,
task.autotasktype.value,
1 if task._is_period else 0,
task.periodnum,
task.cmd.stratagy.interval,
task.laststarttime,
task.lastendtime,
task.cmdstatus.value,
task.batchtotalcount,
task.batchcompletecount,
0,
task.source,
'',
task.createtime,
helper_time.ts_since_1970_tz(),
0,
))
if result is None or result.rowcount < 1: # or len(result) < 1:
res = False
else:
res = True
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
except Exception:
self._logger.error(
"save new AutomatedTask error: %s" % traceback.format_exc())
return res