def parse_conf_task(self, parse_env_video, channel_conf, index=0):
"""
:param parse_env_video, 上承video模块，获取login的token、视频服务器的ID
:param channel_conf, 全局相机唯一配置，通过YML的index索引（键），获取内容，进行解析，得到相机ID
"""
_task_type = channel_conf.split('_')[0].capitalize()
_channel_dict = config_channel.get(channel_conf, index)
_video_obj = parse_env_video(_channel_dict['server_name'])
self.login = _video_obj.login
self.flag = index
self.tasks_type = _task_type if _task_type == 'Crowd' else 'CrossLine'
self.task_cover = os.path.join(DATA_PATH, 'covers',
_channel_dict['task_cover'])
self.tasks_conf = config_task.get(_channel_dict['task_conf'], index)
self.tasks_conf["taskName"] = _channel_dict['task_name']
self.tasks_conf["channelName"] = _channel_dict['channel_name']
mysql = Sql()
_channel_name = self.tasks_conf["channelName"]
query_camera_id = "SELECT F_ID FROM t_video_channel " \
"WHERE F_Name = '{}' " \
"AND F_Video_Server_ID = '{}';".format(_channel_name, _video_obj.server_id)
req = mysql.query(query_camera_id)
mysql.close()
if req is None:
raise ValueError('错误：相机{}不存在'.format(_channel_name))
self.channelId = self.tasks_conf["channelId"] = req[0][0]
logger.debug('获取相机"{}"的ID为{}'.format(_channel_name, self.channelId))
return
def save_weibo_cotent(self):
global last_query_id
weibo_content_save_path = self.data_save_path.format(self.labelids[0])
print(weibo_content_save_path)
sql_obejct = Sql(self.tables[0], self.labelids[0], last_query_id)
sql_weibo_count = sql_obejct.query_weibo_count()
sql_weibo_content = sql_obejct.query_weibo_content()
print(sql_weibo_count)
print(sql_weibo_content)
weibo_count = 0
weibo_content = ()
with ConnectionPool() as cp:
try:
cp.cursor.execute(sql_weibo_count)
temp = cp.cursor.fetchone()
if temp[0] is None:
weibo_count = 0
else:
weibo_count = temp[0]
print(weibo_count)
cp.cursor.execute(sql_weibo_content)
weibo_content = cp.cursor.fetchall()
except Exception as e:
print(e)
if weibo_count != 0:
last_query_id = weibo_count
# 去除文本中的emoji
emoji_pattern = re.compile(
'['
u'\U0001F600-\U0001F64F' # emoticons
u'\U0001F300-\U0001F5FF' # symbols & pictographs
u'\U0001F680-\U0001F6FF' # transport & map symbols
u'\U0001F1E0-\U0001F1FF' # flags (iOS)
']+',
flags=re.UNICODE)
# 去除weibo文本中前面的##部分
pattern = re.compile("#[a-zA-Z0-9\u4E00-\u9FA5]+#")
with open(weibo_content_save_path, 'w+') as f:
if weibo_content:
# 去除文本中的特殊字符<200b><200c><200d>
temp = []
for item in weibo_content:
sub_result = item[0].replace(u'\u200b', '')
sub_result = emoji_pattern.sub(r'', sub_result)
sub_result = pattern.sub(r'', sub_result)
temp.append(sub_result)
f.write('\n'.join(temp))
print('done')
else:
print('weibo_crawler表的该列为空')
def load_video_data():
sql = Sql()
logger.info("载入SQL数据中……")
sql.load_sql(abs_sql_video_server)
sql.load_sql(abs_sql_video_group)
sql.load_sql(abs_sql_video_channel)
logger.info("载入完成")
sql.close()
def query_video_id(video_conf):
mysql = Sql()
if video_conf['version'] == 'rtsp':
sql_video_id = "SELECT F_ID FROM t_video_channel WHERE F_Name = '{}' AND " \
"F_Video_Type = 'rtsp' AND F_Enabled = 1;".format(video_conf['name'])
else:
sql_video_id = "SELECT F_ID FROM t_video_server WHERE F_NAME = '{}' AND " \
"F_Video_Type = 'pvg' AND F_Enabled = 1;".format(video_conf['name'])
req = mysql.query(sql_video_id)
mysql.close()
server_id = req[0][0] if req else None
logger.debug('获取 {} ID: {}'.format(video_conf['name'], server_id))
return server_id
def load_user_data():
sql = Sql()
logger.info("载入SQL数据中……")
sql.load_sql(abs_sql_role_model)
sql.load_sql(abs_sql_user)
logger.info("载入完成")
sql.close()
def load_group_data():
sql = Sql()
logger.info("载入SQL数据中……")
sql.load_sql(abs_sql_custom_group)
sql.load_sql(abs_sql_custom_cameras_group)
logger.info("载入完成")
sql.close()
def assert_server_count(self):
# 通过数据库确认camera数量
query_camera_count = "SELECT COUNT(*) FROM t_video_channel " \
"WHERE F_Video_Server_ID = '{}' " \
"AND F_Enabled = 1;".format(self.server_id)
# logger.debug(query_camera_count)
# 循环检查并断言camera数量
init = -1
start = datetime.datetime.now()
while True:
count = Sql().query(query_camera_count)[0][0]
now = datetime.datetime.now()
interval = (now - start).seconds
if count == self.video_conf['real_count']:
logger.info('查询完毕，结果{}匹配'.format(count))
res = 'SUCCESS'
break
elif count == init and interval > 30:
logger.error('查询完毕，结果不匹配')
res = 'FAIL'
break
else:
init = count
logger.info('查询中，已耗时{}，已同步{}条'.format((now - start), count))
time.sleep(5)
end = datetime.datetime.now()
logger.info('总耗时:{}'.format(end - start))
assert res == 'SUCCESS'
def process_item(self, item, spider):
sql = Sql()
if isinstance(item, NovelsItem):
sql.save_novel(item, spider)
elif isinstance(item, ChaptersItem):
sql.save_chapter(item, spider)
return item
def __init__(self, settings_file, logfile, *, loop=None):
# Load settings
loop = asyncio.get_event_loop() if loop is None else loop
self.settings = Settings(settings_file, loop=loop)
help_name = self.settings.help_name
disabled_cogs = self.settings.disabled_cogs
super().__init__(_command_prefix,
case_insensitive=True,
help_attrs={'name': help_name},
loop=loop)
# Set up logger
self.logger.setLevel(
logging.DEBUG if self.settings.debug else logging.INFO)
handler = logging.FileHandler(logfile, mode='w')
fmt = logging.Formatter(
'%(asctime)s (PID:%(process)s) - %(levelname)s - %(message)s')
handler.setFormatter(fmt)
self.logger.addHandler(handler)
# Load cogs
dname = os.path.dirname(__file__) or '.'
for cogfile in glob.glob(f'{dname}/../cogs/*.py'):
if os.path.isfile(cogfile) and '__init__' not in cogfile:
extn = f'cogs.{os.path.splitext(os.path.basename(cogfile))[0]}'
if extn.split('.')[1] not in disabled_cogs:
try:
self.load_extension(extn)
except discord.ClientException:
self.logger.warning(f'Failed to load cog "{extn}"')
else:
self.logger.info(f'Loaded cog "{extn}"')
else:
self.logger.info(f'Skipping disabled cog "{extn}"')
# Set up sql database
self.sql = Sql()
with self.sql:
self.sql.db_init()
def update_db(self):
"""更新数据库"""
_Sql = Sql()
if self.is_receipt:
d = {
# "QpSeqNo": self.cusCiqNo,
"QpNotes": self.note,
"QpEntryId": self.entryId,
"ProcessTime":
datetime.datetime.now().strftime("%Y%m%d%H%M%S"),
"DecState": "CR_" + self.channel if self.channel else None,
}
import copy
_d = copy.deepcopy(d)
for k in _d:
if _d[k] is None:
d.pop(k)
_Sql.update("NRelation", where={"QpSeqNo": self.cusCiqNo}, **d)
return True
elif self.is_other:
Nids = _Sql.select('NRelation',
'id',
where={"QpSeqNo": getattr(self, "n_SeqNo")})
if Nids:
for nid in Nids:
DecIds = _Sql.select('Relation',
'DecId',
'PId',
where={"NId": nid})
for decid, PId in DecIds:
_Sql.insert('DecLicenseDoc',
DecId=decid,
DocuCode='a',
CertCode=self.n_QpEntryId)
_Sql.update('DecHead',
where={'DecId': decid},
SeqNo=self.dec_SeqNo)
_Sql.update('DecMsg',
where={'DecId': decid},
QpSeqNo=self.dec_SeqNo)
_Sql.update('PassPortHead',
where={'PId': PId},
RltNo=self.n_QpEntryId)
passports = _Sql.select('PassPortHead',
'BindTypecd',
'RltTbTypecd',
'id',
where={'PId': PId})
for passport in passports:
if 2 == passport[0]:
_Sql.insert('PassPortAcmp',
RltTbTypecd=passport[1],
RltNo=self.n_QpEntryId,
Acmp2Head=passport[2])
else:
logger.warning('更新回执失败，核注单统一编号{}未存在'.format(self.n_SeqNo))
return False
return True
else:
d = {
"QpSeqNo": self.SeqNo,
"QpNotes": self.ErrorMessage,
"DecState": "TS_O&K",
}
if d['QpSeqNo'] is None:
d.pop('QpSeqNo')
if d['QpNotes'] and len(d['QpNotes']) > 200:
d['QpNotes'] = d['QpNotes'][:200]
try:
if self.file_name.startswith(
('Successed_PassPort', 'Failed_PassPort')):
Pid = _Sql.select(
'PRelation',
'id',
where={"ClientSeqNo": getattr(self, "ClientSeqNo")})
if Pid:
_Sql.update("PRelation",
where={
"ClientSeqNo":
getattr(self, "ClientSeqNo")
},
**d)
else:
logger.warning("核放单更新回执失败, 自编号{}不存在".format(
self.ClientSeqNo))
return False
else:
Nid = _Sql.select(
'NRelation',
'id',
where={"ClientSeqNo": getattr(self, "ClientSeqNo")})
if Nid:
_Sql.update("NRelation",
where={
"ClientSeqNo":
getattr(self, "ClientSeqNo")
},
**d)
if self.SeqNo:
for NId in Nid:
_Sql.update("NemsInvtHeadType",
where={"NId": NId},
SeqNo=self.SeqNo)
else:
logger.warning("核注清单更新回执失败, 自编号{}不存在".format(
self.ClientSeqNo))
return False
logger.info("更新DecMsg信息,ClientSeqNo:{},msg:{}".format(
getattr(self, "ClientSeqNo"), d))
except Exception as e:
logger.warning('更新回执失败，错误信息：{}'.format(e))
return False
return True
class PikalaxBOT(LoggingMixin, commands.AutoShardedBot):
def __init__(self, settings_file, logfile, *, loop=None):
# Load settings
loop = asyncio.get_event_loop() if loop is None else loop
self.settings = Settings(settings_file, loop=loop)
help_name = self.settings.help_name
disabled_cogs = self.settings.disabled_cogs
super().__init__(_command_prefix,
case_insensitive=True,
help_attrs={'name': help_name},
loop=loop)
# Set up logger
self.logger.setLevel(
logging.DEBUG if self.settings.debug else logging.INFO)
handler = logging.FileHandler(logfile, mode='w')
fmt = logging.Formatter(
'%(asctime)s (PID:%(process)s) - %(levelname)s - %(message)s')
handler.setFormatter(fmt)
self.logger.addHandler(handler)
# Load cogs
dname = os.path.dirname(__file__) or '.'
for cogfile in glob.glob(f'{dname}/../cogs/*.py'):
if os.path.isfile(cogfile) and '__init__' not in cogfile:
extn = f'cogs.{os.path.splitext(os.path.basename(cogfile))[0]}'
if extn.split('.')[1] not in disabled_cogs:
try:
self.load_extension(extn)
except discord.ClientException:
self.logger.warning(f'Failed to load cog "{extn}"')
else:
self.logger.info(f'Loaded cog "{extn}"')
else:
self.logger.info(f'Skipping disabled cog "{extn}"')
# Set up sql database
self.sql = Sql()
with self.sql:
self.sql.db_init()
def run(self):
self.logger.info('Starting bot')
token = self.settings.token or input('Bot OAUTH2 token: ')
super().run(token)
async def close(self):
await super().close()
with self.sql:
self.sql.backup_db()
def get_any_emoji(self, **attrs):
emojis = [emoji for emoji in guild.emojis for guild in self.guilds]
return discord.utils.get(emojis, **attrs)
@property
def command_error_emoji(self):
return self.get_emoji(356887966947344384)
def cmd_error_check(self, ctx, exc):
if isinstance(exc, commands.CommandNotFound):
return False
# Inherit checks from super
if self.extra_events.get('on_command_error', None):
self.log_and_print(logging.DEBUG,
'on_command_error in extra_events')
return False
if hasattr(ctx.command, 'on_error'):
self.log_and_print(logging.DEBUG, f'{ctx.command} has on_error')
return False
cog = ctx.cog
if cog:
attr = f'_{cog.__class__.__name__}__error'
if hasattr(cog, attr):
self.log_and_print(logging.DEBUG,
f'{cog.__class__.__name__} has __error')
return False
return True
async def on_command_error(self, ctx: commands.Context, exc):
async def report(msg):
debug = self.settings.debug
await ctx.send(f'{ctx.author.mention}: {msg} {emoji}',
delete_after=None if debug else 10)
if debug:
self.log_tb(ctx, exc)
if not self.cmd_error_check(ctx, exc):
return
emoji = self.command_error_emoji
if isinstance(exc,
commands.NotOwner) and ctx.command.name != 'pikahelp':
await report('Permission denied')
elif isinstance(exc, commands.MissingPermissions):
await report(
f'You are missing permissions: {", ".join(exc.missing_perms)}')
elif isinstance(exc, commands.BotMissingPermissions):
await report(
f'I am missing permissions: {", ".join(exc.missing_perms)}')
elif exc is NotImplemented:
await report(
'The command or one of its dependencies is not fully implemented'
)
elif isinstance(exc, commands.UserInputError):
await report(f'**{type(exc).__name__}**: {exc}')
elif isinstance(exc, commands.CheckFailure):
return
else:
self.log_tb(ctx, exc)
async def _before_invoke(self, ctx):
ctx.cog.fetch()
if self.settings.debug:
await self.owner.send(
f'before_invoke: {ctx.cog.__class__.__name__}.{ctx.command}')
async def _after_invoke(self, ctx):
ctx.cog.commit()
if self.settings.debug:
await self.owner.send(
f'after_invoke: {ctx.cog.__class__.__name__}.{ctx.command}')
def update_db(self):
"""更新数据库"""
_Sql = Sql()
if self.is_receipt:
# d = {
# # "QpSeqNo": self.cusCiqNo,
# "QpNotes": self.note,
# "QpEntryId": self.entryId,
# "ProcessTime": datetime.datetime.now().strftime("%Y%m%d%H%M%S"),
# "DecState": "CR_" + self.channel,
#
# }
# if d["QpEntryId"] is None:
# d.pop("DecDate")
# _Sql.update("DecMsg", where={"QpSeqNo": self.cusCiqNo}, **d)
# # logger.info("更新DecMsg信息,ClientSeqNo:{},msg:{}".format(getattr(self, "ClientSeqNo"), d))
#
# # 插入DecReceipt信息
# ret = _Sql.select("DecMsg", "DecId,ClientSeqNo", where={"QpSeqNo": self.cusCiqNo})
# if not ret:
# logger.warn(
# "插入数据到DecReceipt表时，根据统一编号在DecMsg搜索DecId，ClientSeqNo,未搜到，严重逻辑错误，" +
# "说明申报成功后应该将QpSeqNo更新到DecMsg，但此步未做，本DecReceipt信息：{}, 'QpSeqNo': {}".format(d, self.cusCiqNo))
# return False
#
# DecId, ClientSeqNo = ret[0]
# d = {
# "DecId": DecId,
# "SeqNo": self.cusCiqNo,
# "ClientSeqNo": ClientSeqNo,
# "NoticeDate": self.noticeDate,
# "DecState": "CR_" + self.channel,
# "Note": self.note,
# "DecDate": self.dDate, # 申报日期
# "IEDate": self.dDate, # 进出口日期
# }
# if self.dDate:
# pass
# else:
# d.pop("DecDate")
# d.pop("IEDate")
#
# _Sql.insert("DecReceipt", **d)
# logger.info("单一窗口：报关单海关回执写入数据库DecReceipt成功:{}".format(d))
return True
else:
d = {
"QpSeqNo": self.SeqNo,
"QpNotes": self.ErrorMessage,
"DecState": "TS_O&K",
}
if self.file_name.startswith('Failed'):
d.pop('QpSeqNo')
if d['QpNotes'] and len(d['QpNotes']) > 200:
d['QpNotes'] = d['QpNotes'][:200]
try:
_Sql.update("NRelation", where={"ClientSeqNo": getattr(self, "ClientSeqNo")}, **d)
logger.info("更新DecMsg信息,ClientSeqNo:{},msg:{}".format(getattr(self, "ClientSeqNo"), d))
except Exception as e:
logger.warning('更新回执失败，错误信息：{}'.format(e))
return False
return True
def update_db(self):
"""更新数据库"""
_Sql = Sql()
if self.is_receipt:
d = {
# "QpSeqNo": self.cusCiqNo,
"QpNotes": self.note,
"QpEntryId": self.entryId,
"ProcessTime": datetime.datetime.now().strftime("%Y%m%d%H%M%S"),
"DecState": "CR_" + self.channel,
}
_Sql.update("DecMsg", where={"QpSeqNo": self.cusCiqNo}, **d)
# logger.info("更新DecMsg信息,ClientSeqNo:{},msg:{}".format(getattr(self, "ClientSeqNo"), d))
# 插入DecReceipt信息
ret = _Sql.select("DecMsg", "DecId,ClientSeqNo", where={"QpSeqNo": self.cusCiqNo})
if not ret:
logger.warn(
"插入数据到DecReceipt表时，根据统一编号在DecMsg搜索DecId，ClientSeqNo,未搜到，严重逻辑错误，"+
"说明申报成功后应该将QpSeqNo更新到DecMsg，但此步未做，本DecReceipt信息：{}, 'QpSeqNo': {}".format(d, self.cusCiqNo))
return False
DecId,ClientSeqNo= ret[0]
d = {
"DecId": DecId,
"SeqNo": self.cusCiqNo,
"ClientSeqNo": ClientSeqNo,
"NoticeDate": self.noticeDate,
"DecState": "CR_" + self.channel,
"Note": self.note,
"DecDate": self.dDate, # 申报日期
"IEDate": self.dDate, # 进出口日期
}
if self.dDate:
pass
else:
d.pop("DecDate")
d.pop("IEDate")
_Sql.insert("DecReceipt", **d)
logger.info("单一窗口：报关单海关回执写入数据库DecReceipt成功:{}".format(d))
return True
else:
# if self.failInfo:
# d = {
# "QpNotes": self.failInfo,
# "ProcessTime": datetime.datetime.now().strftime("%Y%m%d%H%M%S"),
#
# }
# _Sql.update("DecMsg", where={"ClientSeqNo": getattr(self, "ClientSeqNo")}, **d)
# logger.info("更新DecMsg,回执中包含错误信息,ClientSeqNo:{},msg:{}".format(getattr(self, "ClientSeqNo"), d))
# return True
# elif self.retData:
# self.retData = json.loads(self.retData)
# responseMessage = self.retData.get("responseMessage")
# responseCode = self.retData.get("responseCode")
# cusCiqNo = self.retData.get("cusCiqNo")
# preEntryId = self.retData.get("preEntryId")
#
# d = {
# "QpSeqNo": cusCiqNo,
# "QpNotes": responseMessage,
# "QpEntryId": preEntryId,
# "ProcessTime": datetime.datetime.now().strftime("%Y%m%d%H%M%S"),
# "DecState": "TS_REQ",
# }
# _Sql.update("DecMsg", where={"ClientSeqNo": getattr(self, "ClientSeqNo")}, **d)
# logger.info("更新DecMsg信息,ClientSeqNo:{},msg:{}".format(getattr(self, "ClientSeqNo"), d))
# return True
d = {
"QpSeqNo": self.SeqNo,
"QpNotes": self.ErrorMessage,
}
if d['QpSeqNo'] is None:
d.pop('QpSeqNo')
if d['QpNotes'] and len(d['QpNotes']) > 200:
d['QpNotes'] = d['QpNotes'][:200]
if "0" == self.failInfo:
d['DecState'] = 'TS_O&K'
else:
d['DecState'] = 'TS_ERR'
decid_tuple = _Sql.select("DecMsg", 'DecId', where={"ClientSeqNo": getattr(self, "ClientSeqNo")})
try:
if decid_tuple:
_Sql.update("DecMsg", where={"ClientSeqNo": getattr(self, "ClientSeqNo")}, **d)
logger.info("更新DecMsg信息,ClientSeqNo:{},msg:{}".format(getattr(self, "ClientSeqNo"), d))
else:
logger.warning("更新回执失败, 自编号{}不存在".format(self.ClientSeqNo))
return False
except Exception as e:
logger.warning('更新回执失败，错误信息：{}'.format(e))
return False
return True
def clear_all():
sql = Sql()
sql.load_sql(abs_sql_init_all)
sql.close()
def load_task_data():
sql = Sql()
logger.info("载入SQL数据中……")
sql.load_sql(abs_sql_task) # 人群任务的前后端同步机制，造成这里导入数据之后，任务的默认状态只能是fail，需要间隔一会儿自动变成run
logger.info("载入完成")
sql.close()