def update_cmd(
self,
cmd: IdownCmd,
taskid: str = None,
batchid: str = None,
isrelative: int = 1,
) -> bool:
"""更新cmd，条件为platform+cmdid，更新其他可更新字段"""
conn: SqliteConn = None
cursor = None
res: bool = False
cmd: IdownCmd = cmd
try:
# ClientId
# CmdId,
# Platform,
# TaskId,
# BatchId,
# Cmd,
# Status,
# Source,
# CmdRcvMsg,
# CreateTime,
# UpdateTime,
# Sequence,
# IsRelative
cmd = f'''UPDATE {self._tbname} set
TaskId=?,
BatchId=?,
Cmd=?,
Status=?,
Source=?,
CmdRcvMsg=?,
CreateTime=?,
UpdateTime=?,
Sequence=?,
IsRelative=?
WHERE Platform=? and CmdId=?;'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
# task:Task = task
result = cursor.execute(
cmd,
(
taskid,
batchid,
cmd.cmd_str,
cmd.cmdstatus.value,
cmd.source,
cmd.cmdrcvmsg,
# 假设数据是按时间先后顺序过来的，所以createtime直接取本地时间。
# 若后面有问题，需要加idowncmd命令文件字段createtime字段
helper_time.get_time_sec_tz(),
helper_time.get_time_sec_tz(),
cmd._sequence,
isrelative,
cmd._platform,
cmd.cmd_id,
))
if not result is None and result.rowcount > 0:
res = True
break
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
except Exception:
self._logger.error("update {} error: {}".format(
self._tbname, traceback.format_exc()))
return res
def update_batchtask_back(self, tb: TaskBatchBack) -> bool:
"""使用指定 TaskBatchBack 对象更新IDownBatchTask表，
不更新 isbatchcompletecountincreased\n"""
conn: SqliteConn = None
cursor = None
res: bool = False
tb: TaskBatchBack = tb
try:
if not isinstance(tb, TaskBatchBack):
self._logger.error(
"Invalid param TaskBatchBack: {}".format(tb))
return res
# Platform
# TaskId,
# Batchid,
# ClientId,
# TokenId,
# AppType,
# CmdRcvMsg,
# Result,
# CmdStatus,
# Progress,
# Sequence,
# OtherFields,
# CreateTime,
# UpdateTime,
# IsBatchCompleteCountIncreased
# CmdId
# 更新策略，先搜一下有没有，并把sequence搜出来，如果
# 本地sequence和新来的任务的sequence一样，则说明
# 采集端的sequence出错了，打一句日志并返回False
cmd = f'''SELECT Sequence FROM {self._tbname} WHERE Platform=? and TaskId=? and BatchId=?;'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
cursor.execute(cmd, (
tb._platform,
tb._taskid,
tb._batchid,
))
result = cursor.fetchall()
if result is None or len(result) < 1:
continue
oldseq = result[0][0]
if oldseq >= tb._sequence:
self._logger.error(
"The comming TaskBatchBack.sequence is {}, but which in local db is {}:\ntaskid:{}\nbatchid:{}\nsequence:{}"
.format(tb._sequence, oldseq, tb._taskid,
tb._batchid, oldseq))
break
cmd = f'''UPDATE {self._tbname} set
CmdRcvMsg=?,
Result=?,
CmdStatus=?,
Progress=?,
Sequence=?,
UpdateTime=?
WHERE Platform=? and TaskId=? and BatchId=? and Sequence<?;'''
cursor = conn.cursor
result = cursor.execute(cmd, (
tb._cmdrcvmsg,
tb._result,
tb._cmdstatus.value,
tb._progress,
tb._sequence,
helper_time.get_time_sec_tz(),
tb._platform,
tb._taskid,
tb._batchid,
tb._sequence,
))
if not result is None and result.rowcount > 0:
res = True # 一定只有一个子任务
break
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
except Exception:
self._logger.error("update {} error: {}".format(
self._tbname, traceback.format_exc()))
return res
def update_batchtask(self, task: Task) -> bool:
"""使用指定task对象更新IDownBatchTask表，
不更新 isbatchcompletecountincreased\n
task: 表示一个子任务对象"""
conn: SqliteConn = None
cursor = None
res: bool = False
try:
task._cmdstatus
cmd = f'''UPDATE {self._tbname} set
ClientId=?,
ParentBatchId=?,
TokenId=?,
AppType=?,
ForceDownload=?,
CmdRcvMsg=?,
Result=?,
CmdStatus=?,
Progress=?,
Sequence=?,
OtherFields=?,
CreateTime=?,
UpdateTime=?,
Source=?,
CmdId=?
WHERE Platform=? and TaskId=? and BatchId=?;'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
# task:Task = task
result = cursor.execute(cmd, (
task._clientid,
task.parentbatchid,
task.tokenid,
task.apptype,
task.forcedownload,
task._cmdrcvmsg,
task._result,
task._cmdstatus.value,
task.progress,
task._sequence,
task.other_fields_json,
task.timestr,
helper_time.get_time_sec_tz(),
task.source,
task.cmd_id,
task._platform,
task.taskid,
task.batchid,
))
if not result is None and result.rowcount > 0:
res = True # 一定只有一个子任务
break
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
except Exception:
self._logger.error("update {} error: {}".format(
self._tbname, traceback.format_exc()))
return res
def save_new_idownbatchtask(
self,
task: Task,
client: Client,
cmdstatus: ECommandStatus = ECommandStatus.WaitForSend
) -> Tuple[bool, bool]:
"""保存新的批处理任务的子任务\n
task: 任务对象\n
tokenid：当前批处理子任务对应的登陆令牌存储在IDownBatchTokens表中的tokenid，关联存储\n
client: 当前批处理子任务被分配到的client\n
cmdstatus: 指定当前批处理子任务的命令状态\n
return: （bool是否成功，bool是否为新增(用于计算batchtotalcount)）"""
res: bool = False
isnew: bool = False
conn: SqliteConn = None
cursor = None
client: Client = client
task: Task = task
try:
if not isinstance(task.tokenid, str) or task.tokenid == "":
raise Exception(
"Invalid tokenid while save new idownbatchtask:\ntaskid:{}"
.format(task.taskid))
# 搜索每个库，看有没有 taskid和clientid一样，且时间更新
# 的，一样就更新其他所有字段
cmd = f'''SELECT COUNT(1) FROM {self._tbname} WHERE
Platform=? and TaskId=? and BatchId=? and ClientId=?'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
cursor.execute(cmd, (
task._platform,
task.taskid,
task.batchid,
client._statusbasic._clientid,
))
result = cursor.fetchall()
if result[0][0] > 0:
res = True
# 只根据TaskId、ParentTaskId和ClientId关联，
# 不考虑 任务文件产生时间与现有数据库中已存在任务的时间，每次直接覆盖，以符合用户操作。
# 若来了TaskId一样的数据，则必然分配给同一个ClientId
cmd = f'''UPDATE {self._tbname} set
TokenId=?,
AppType=?,
ForceDownload=?,
CmdRcvMsg=?,
Result=?,
CmdStatus=?,
Progress=?,
Sequence=?,
OtherFields=?,
CreateTime=?,
UpdateTime=?,
Source=?,
CmdId=?
WHERE Platform=? and TaskId=? and BatchId=? and ClientId=? and CreateTime<=?;'''
result = cursor.execute(
cmd,
(
task.tokenid,
task.apptype,
task.forcedownload,
task._cmdrcvmsg,
task._result,
cmdstatus.value,
# 进度置零
0,
#批量任务完成数也要重置，因为重置了任务下发状态，变为新任务，需要重新执行
0, #sequence重置
task.other_fields_json,
task.timestr,
helper_time.get_time_sec_tz(),
task.source,
task.cmd_id,
# 0 此字段在其他地方重置 #是否总任务已提升batchcompletecount置零，
task._platform,
task.taskid,
task.batchid,
client._statusbasic._clientid,
task.timestr,
))
# 这句没用，就是调试看看结果..
if result is None or result.rowcount < 1: # or len(result) < 1:
pass
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
# 若没找到，则insert一条到最新的库
# res==True表示至少有一个库里面有一条符合条件的任务，且已更新其字段
if not res:
isnew = True
conn = self.connect_write(5)
try:
# insert
# Platform,
# TaskId,
# Batchid,
# ParentBatchId,
# ClientId,
# TokenId,
# AppType,
# ForceDownload,
# CmdRcvMsg,
# Result,
# CmdStatus,
# Progress,
# Sequence,
# OtherFields,
# CreateTime,
# UpdateTime
# IsBatchCompleteCountIncreased
cmd = f'''INSERT INTO {self._tbname}(
Platform,
TaskId,
Batchid,
ParentBatchId,
ClientId,
TokenId,
AppType,
ForceDownload,
CmdRcvMsg,
Result,
CmdStatus,
Progress,
Sequence,
OtherFields,
CreateTime,
UpdateTime,
Source,
CmdId) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'''
# 有几个属于TaskBack的字段在此不更新
cursor = conn.cursor
result = cursor.execute(
cmd,
(
task._platform,
task.taskid,
task.batchid,
task.parentbatchid,
client._statusbasic._clientid,
task.tokenid,
task.apptype,
task.forcedownload,
task._cmdrcvmsg,
task._result,
cmdstatus.value,
0, # progress初始化为0
0, #sequence初始化为0
task.other_fields_json,
task.timestr,
helper_time.get_time_sec_tz(),
task.source,
task.cmd_id))
if result is None or result.rowcount < 1: # or len(result) < 1:
res = False
else:
res = True
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
except Exception:
self._logger.error("save_new_idownbatchtask error: %s" %
traceback.format_exc())
return (res, isnew)
def __init__(self, allfields: dict):
"""使用必要值实例化一个Task任务对象"""
if not isinstance(allfields, dict) or len(allfields) < 1:
raise Exception("Task initial fields is invalid")
# 所有字段集合(包括数据中带过来的其他需要回传的字段)
# 输出数据时根据所属平台，需要输出的额外字段，都在这里面来找
# 比如：有一个邮件数据EMail，输出到某个标准时需要一个casenode字段
# 这个字段直接在 EMail.Task._allfields.get("casenode")里找
# 统一使用self._judge()，不使用allfields.get()
self._allfields: dict = allfields
# 若allfields是从数据库读出来，则含有otherfields的json字符串，需要
# 解析并添加到allfields中，方便下面解析字段
# 指示是否强制使用指定的tokentype进行登陆。
tmpotherstr = self._judge(allfields, "otherfields")
self._parse_otherfields(tmpotherstr)
# self._other_fields.pop("otherfields", None)
# 其他字段，用于存储除以下标准字段以外的，带在数据文件里的其他字段
# 在 self._judge()方法中会自动剔除标准字段
self._other_fields: dict = allfields.copy()
# 当前数据产生时间，必要，默认的时间都改成东8区
self._time: float = datetime.now(
pytz.timezone("Asia/Shanghai")).timestamp()
# 这个platform不知道哪些地方在用，先留着
self.platform: str = self._judge(allfields,
"platform",
error=True,
excludefields=self._other_fields)
self._source: str = self._judge(allfields,
"source",
error=False,
excludefields=self._other_fields)
OutputData.__init__(self, self.platform, EStandardDataType.Task)
OutputDataSeg.__init__(self)
# 必要字段(数据标准字段)
self.taskid: str = self._judge(allfields,
"taskid",
error=True,
excludefields=self._other_fields)
# 子任务id，对控制端为非必要，对采集端为必要
self.batchid: str = self._judge(allfields,
"batchid",
dft="",
error=False,
excludefields=self._other_fields)
# 登陆令牌tokenid，需要与界面同步。也方便后面做单机版
self.tokenid: str = self._judge(
allfields,
"tokenid",
# 这个字段没有的，说明不是中心下发的任务，就要随机一个tokenid
# 且在解析时就判断了如果是中心下发的任务，若无此字段，就会报错
dft=str(uuid.uuid1()),
error=False,
excludefields=self._other_fields,
)
# 非必要字段(数据标准字段)
# 关联任务的taskid
self.parenttaskid: str = self._judge(allfields,
"parenttaskid",
excludefields=self._other_fields)
# 关联的父子任务
self.parentbatchid: str = self._judge(allfields,
"parentbatchid",
excludefields=self._other_fields)
# 状态字段 (非数据标准字段)
self.taskstatus: ETaskStatus = ETaskStatus(
int(
self._judge(
allfields,
"taskstatus",
ETaskStatus.New.value,
excludefields=self._other_fields,
))) # 任务当前状态
# 批量任务的子任务总数
self.batchtotalcount: int = self._judge(
allfields, "batchtotalcount", 0, excludefields=self._other_fields)
# 批量任务的已完成的任务数
self.batchcompletecount: int = self._judge(
allfields,
"batchcompletecount",
0,
excludefields=self._other_fields)
self.isbatchcompletecountincreased: bool = self._judge(
allfields,
"isbatchcompletecountincreased",
False,
excludefields=self._other_fields,
)
# 任务类型
self.tasktype: ETaskType = ETaskType(
int(
self._judge(allfields,
"tasktype",
error=True,
excludefields=self._other_fields)))
# 登陆资源/令牌类型
self.tokentype: ETokenType = ETokenType(
int(
self._judge(
allfields,
"tokentype",
dft=ETokenType.Unknown.value,
error=False,
excludefields=self._other_fields,
)))
# 必须有apptype，因为任务文件在server解出来，
# 就是一个一个的子任务了
self.apptype: int = int(
self._judge(
allfields,
"apptype",
dft=0,
error=False,
excludefields=self._other_fields,
))
# 交互输入（验证码等）
self.input: str = self._judge(allfields,
"input",
excludefields=self._other_fields)
# 内部预置国际号码标识
self.preglobaltelcode: str = self._judge(
allfields, "preglobaltelcode", excludefields=self._other_fields)
# 内部预置账号
self.preaccount: str = self._judge(allfields,
"preaccount",
excludefields=self._other_fields)
# 资源字段(数据标准字段)
self.globaltelcode: str = self._judge(allfields,
"globaltelcode",
excludefields=self._other_fields)
self.phone: str = self._judge(allfields,
"phone",
excludefields=self._other_fields)
self.account: str = self._judge(allfields,
"account",
excludefields=self._other_fields)
self.password: str = self._judge(allfields,
"password",
excludefields=self._other_fields)
self.url: str = self._judge(allfields,
"url",
excludefields=self._other_fields)
self.host: str = self._judge(allfields,
"host",
excludefields=self._other_fields)
self.cookie: str = self._judge(allfields,
"cookie",
excludefields=self._other_fields)
# 新加字段，是否强制下载
self._forcedownload: str = self._judge(
allfields,
"forcedownload",
dft=False,
excludefields=self._other_fields)
# 任务文件过来时的时间，时间戳转化一下
self.timestr: str = None # 日期时间格式字符串
if self.time is not None and type(self.time) in [int, float]:
self.timestr = datetime.fromtimestamp(
self.time,
pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
if self.timestr is None:
self.timestr = helper_time.get_time_sec_tz()
# 这里新任务来了应该是没有time字段的
if self._allfields.__contains__("time"):
self._allfields["time"] = self.timestr
# 任务回馈相关字段，在TaskBack类型中实现
# 这两个时间，应该是写入数据库当时，获取的当前时间，而不是从任务文件里来的
self.createtime = self._judge(
allfields,
"createtime",
dft=int(self._time),
excludefields=self._other_fields,
)
self.updatetime = self._judge(allfields,
"updatetime",
excludefields=self._other_fields)
# 任务进度信息
# 当Task作为子任务时，有progress字段
# 当Task作为总任务时，progress通过计算得到
self._progress: float = self._judge(allfields,
"progress",
dft=0,
error=False,
excludefields=self._other_fields)
self.forcetokentype: bool = False
# 控制端独有
self.deal_succeed: bool = None # 标记当前任务是否分配完成
# 采集端独有
self.lastexecutetime = self._judge(
allfields, "lastexecutetime",
excludefields=self._other_fields) # 最后访问时间, unxitime需要什么类型自己转
self.failtimes = self._judge(allfields,
"failtimes",
0,
excludefields=self._other_fields)
self.successtimes = self._judge(allfields,
"successtimes",
0,
excludefields=self._other_fields)
# 当任务从数据库中读出来时，会带有TaskBack数据的部分字段，需要读取出来，以免被
# 算入self._other_fields里
self._clientid: str = self._judge(allfields,
"clientid",
excludefields=self._other_fields)
self._cmdstatus: ECommandStatus = ECommandStatus(
int(
self._judge(
allfields,
"cmdstatus",
dft=ECommandStatus.WaitForSend.value,
error=False,
excludefields=self._other_fields,
)))
self._cmdrcvmsg: str = self._judge(allfields,
"cmdrcvmsg",
excludefields=self._other_fields)
self._result: str = self._judge(allfields,
"result",
excludefields=self._other_fields)
# tgback文件sequence, 用于回馈数据的文件顺序，要存入数据库
self._sequence: int = self._judge(allfields,
"sequence",
0,
excludefields=self._other_fields)
self._sequence_locker = threading.RLock()
# 通用回调函数
self.on_complete = self._on_complete
# 新增source, 任务来源，需要带回去
self.source = self._judge(allfields,
"source",
self._platform,
excludefields=self._other_fields)
self.casenode: str = self._judge(allfields,
"casenode",
error=False,
excludefields=self._other_fields)
# cmd_id会带在task里
self.cmd_id = self._judge(allfields,
"cmdid",
None,
excludefields=self._other_fields)
self.cmd: IdownCmd = None
# 会在cmd里面拿一个优先级，这里先定义着
self.priority = 2
self.cmdstr = self._judge(allfields,
"cmd",
None,
excludefields=self._other_fields)
if self.cmdstr is not None:
# 任务的命令设置，如果有cmd的话那么会直接补齐使用，如果没有的话那么会使用默认的
self.cmd = IdownCmd(self.cmdstr, self.cmd_id, self.platform,
self._source, self._clientid)
# 新增cookie保活状态和上次cookie保活时间
# 0为失效，1为有效
self.cookie_alive = self._judge(allfields,
"cookiealive",
None,
excludefields=self._other_fields)
self.cookie_last_keep_alive_time = self._judge(
allfields,
"cookielastkeeptime",
None,
excludefields=self._other_fields)
def save_new_iscoutbtask(
self,
task: IscoutTask,
client: Client,
cmdstatus: ECommandStatus = ECommandStatus.WaitForSend,
) -> Tuple[bool, bool]:
"""保存新的批处理任务的令牌资源\n
task:任务对象"""
res = False
isnew: bool = False
conn: SqliteConn = None
cursor = None
task: IscoutTask = task
try:
# 搜索每个库，看有没有 TokenId一样的，且时间更新
# 的，一样就更新其他所有字段
cmd = f'''SELECT COUNT(1) FROM {self._tbname} WHERE
TaskId=? and BatchId=? and Platform=?'''
for conn in self.connect_all(5):
conn: SqliteConn = conn
try:
cursor = conn.cursor
cursor.execute(cmd, (
task.taskid,
task.batchid,
task._platform,
))
result = cursor.fetchall()
if result[0][0] > 0:
res = True
cmd = f'''UPDATE {self._tbname} set
ClientId=?,
CmdId=?,
Source=?,
IsPeriod=?,
PeriodNum=?,
StartTime=?,
EndTime=?,
Interval=?,
LastStartTime=?,
LastEndTime=?,
Status=?,
ObjectType=?,
Object=?,
Progress=?,
Elapsed=?,
CmdRcvMsg=?,
IsBatchCompleteCountIncreased=?,
UpdateTime=?,
CreateTime=?,
Sequence=?
WHERE TaskId=? and BatchId=? and Platform=? and CreateTime<=?;'''
result = cursor.execute(
cmd,
(
client._statusbasic._clientid,
task.cmd_id,
task.source,
1 if task._is_period else 0,
# 这里的update是来了新的任务文件，需要直接覆盖periodnum
task.periodnum,
task.cmd.stratagy.time_start,
task.cmd.stratagy.time_end,
task.cmd.stratagy.interval,
task.laststarttime,
task.lastendtime,
cmdstatus.value,
task._objecttype.value,
task._object,
task.progress,
0, # elapsed
task.cmdrcvmsg,
task.isbatchcompletecountincreased,
helper_time.get_time_sec_tz(),
task.createtimestr,
0, #重置sequence
task.taskid,
task.batchid,
task._platform,
task.createtimestr,
))
# 这句没用，就是调试看看结果..
if result is None or result.rowcount < 1: # or len(result) < 1:
pass
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
if res:
break
# 若没找到，则insert一条到最新的库
# res==True表示至少有一个库里面有一条符合条件的任务，且已更新其字段
if not res:
isnew = True
conn = self.connect_write(5)
try:
# insert
cmd = f'''INSERT INTO {self._tbname}(
ClientId,
Platform,
Source,
IsPeriod,
PeriodNum,
StartTime,
EndTime,
Interval,
LastStartTime,
LastEndTime,
TaskId,
BatchId,
CmdId,
Status,
ObjectType,
Object,
Progress,
Elapsed,
CmdRcvMsg,
IsBatchCompleteCountIncreased,
CreateTime,
UpdateTime,
Sequence) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'''
# 有几个属于TaskBack的字段在此不更新
cursor = conn.cursor
result = cursor.execute(cmd, (
client._statusbasic._clientid,
task._platform,
task.source,
1 if task._is_period else 0,
task.periodnum,
task.cmd.stratagy.time_start,
task.cmd.stratagy.time_end,
task.cmd.stratagy.interval,
task.laststarttime,
task.lastendtime,
task.taskid,
task.batchid,
task.cmd_id,
cmdstatus.value,
task._objecttype.value,
task._object,
task.progress,
0,
'',
task.isbatchcompletecountincreased,
task.createtimestr,
helper_time.get_time_sec_tz(),
0,
))
if result is None or result.rowcount < 1: # or len(result) < 1:
res = False
else:
res = True
except Exception as ex:
conn._conn.rollback()
raise ex
else:
conn.commit()
finally:
if not conn is None:
conn.close()
except Exception:
self._logger.error("save new IScoutTask error: %s" %
traceback.format_exc())
return (res, isnew)