def get_rss_title_and_url(self):
"""
根据规则获取rss的标题和url
:return:
"""
try:
rss_data = json.load(
open(BASE_PATH + "/core/data.json", "r", encoding="utf-8"))
for item in rss_data:
rss = feedparser.parse(requests.get(
item['rss']).content)['entries']
push_new_list = {"source": item, "target": []}
for it in rss:
datetime_struct = parser.parse(it['published'])
published = datetime_struct.strftime("%Y-%m-%d")
today = time.strftime("%Y-%m-%d")
if today == published:
if item['has_content'] in it['title']:
push_new_list["target"].append(it)
self.news_list.append(push_new_list)
except Exception as e:
logger.warn("获取RSS标题和URL异常:" + str(e))
def download_image(self, content):
"""
下载图片
:param content:
:return:
"""
bs = BeautifulSoup(content, 'lxml')
images = bs.find_all("img")
for image in images:
tmpUrl = image.get('src')
(url, suffix) = re.findall(r'(.+?\.(png|jpg|gif|jpeg))', tmpUrl)[0]
try:
if url not in self.picBlacklist:
ir = requests.get(url)
if ir.status_code == 200:
logger.info("获取到图片{picUlr}".format(picUlr=url))
filename = str(uuid.uuid1()) + "." + suffix
open(BASE_PATH + 'static/' + filename,
'wb').write(ir.content)
logger.info(
"下载图片{picName},本地图片名字{localPicName}".format(
picName=url, localPicName=filename))
content.replace(url, self.static_domain + filename)
except Exception as e:
logger.warn("下载图片异常:" + str(e))
return content
def load_same_pinyin(path, sep='\t'):
"""
加载同音字
:param path:
:param sep:
:return:
"""
result = dict()
if not os.path.exists(path):
logger.warn("file not exists:" + path)
return result
with codecs.open(path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if line.startswith('#'):
continue
parts = line.split(sep)
if parts and len(parts) > 2:
key_char = parts[0]
same_pron_same_tone = set(list(parts[1]))
same_pron_diff_tone = set(list(parts[2]))
value = same_pron_same_tone.union(same_pron_diff_tone)
if key_char and value:
result[key_char] = value
return result
def download_image(self, content, source_url):
"""
下载图片
:param source_url:
:param content:
:return:
"""
bs = BeautifulSoup(content, 'lxml')
images = bs.find_all("img")
for image in images:
tmpUrl = image.get('src')
(url, suffix) = re.findall(r'(.+?\.(png|jpg|gif|jpeg))', tmpUrl)[0]
try:
if url not in self.picBlacklist:
if str(url).startswith("http") is False:
url = urljoin(source_url, url)
ir = requests.get(url)
if ir.status_code == 200:
logger.info("Get into the {picUlr}".format(picUlr=url))
filename = str(uuid.uuid1()) + "." + suffix
open(BASE_PATH + 'static/' + filename, 'wb').write(ir.content)
logger.info("Download Pic for {picName},save to local pic name is {localPicName}"
.format(picName=url, localPicName=filename))
content = content.replace(tmpUrl, self.static_domain + filename)
except Exception as e:
logger.warn("下载图片异常:" + str(e))
return content
def article_tag(request, pid):
try:
tag = Tag.objects.get(id=pid)
except Exception as ex:
logger.warn(msg="标签不存在: {ex}".format(ex=ex))
return render(request, 'wiki/wiki_base.html', {
"user": request.user,
"errorInfo": "标签不存在: {ex}".format(ex=ex)
})
categoryList = []
for ds in Category.objects.all():
ds.post_count = Post.objects.filter(category=ds).count()
categoryList.append(ds)
postList = Post.objects.filter(tags=tag.id).order_by('-id')
dateList = Post.objects.raw(
'SELECT id,DATE_FORMAT( created_time, "%%Y/%%m" ) as cdate , COUNT(*) as count FROM opsmanage_wiki_post GROUP BY DATE_FORMAT( created_time, "%%Y/%%m" );'
)
tagList = Tag.objects.all()
return render(
request, 'wiki/wiki_base.html', {
"user": request.user,
"postList": postList,
"dateList": dateList,
"tagList": tagList,
"categoryList": categoryList
})
async def on_ready(self):
'''
Reconstruct poll listeners
'''
coros = []
for poll in collection("polls").find():
logger.info(f"Found poll in channel: {poll['channel']}")
channel: discord.TextChannel = self.bot.get_channel(
poll["channel"])
try:
msg = await channel.fetch_message(poll["_id"])
except:
# msg not found
collection("polls").delete_one(poll)
logger.warn("Message deleted for poll, unloading")
continue
ctx = await self.bot.get_context(msg)
author = msg.guild.get_member(poll["author"])
coros.append(
self.listen_poll(ctx, msg, poll["timeout"], poll["prompt"],
poll["options"], author))
await asyncio.gather(*coros)
def save_post_to_mysql(self, title, source_url, s_from, published_time, start_str, end_str, lazyLoading, lazyLabel):
"""
保存文章到数据库
:param title:
:param source_url:
:param s_from:
:param published_time:
:param start_str:
:param end_str:
:param lazyLoading:
:param lazyLabel:
:return:
"""
content = ''
try:
r = requests.get(source_url).text
content = r[r.find(start_str):r.find(end_str)]
if lazyLoading:
content = content.replace(lazyLabel, "src")
if self.info['images']:
content = self.download_image(content, source_url)
except Exception as e:
r = requests.get(source_url).text
content = r[r.find(start_str):r.find(end_str)]
logger.warn("下载图片出错：" + str(e))
finally:
try:
Mysql().save_to_post(title=title, source_url=source_url, s_from=s_from, content=content,
published_time=published_time, m_hash=self.m_hash)
except Exception as e:
logger.warn("保存文章到数据库出错,错误信息" + str(e))
def update_apps(self,request):
project = self.get_apps(request)
if project:
try:
project.project_env = request.POST.get('project_env')
project.project_type = request.POST.get('project_type')
project.project_repertory = request.POST.get('project_repertory')
project.project_address = request.POST.get('project_address')
project.project_remote_command = request.POST.get('project_remote_command')
project.project_pre_remote_command = request.POST.get('project_pre_remote_command')
project.project_local_command = request.POST.get('project_local_command')
project.project_model = request.POST.get('project_model')
project.project_dir = request.POST.get('project_dir')
project.project_user = request.POST.get('project_user')
project.project_is_include = request.POST.get('project_is_include')
project.project_exclude = request.POST.get('project_exclude')
project.project_repo_user = request.POST.get('project_repo_user')
project.project_repo_passwd = request.POST.get('project_repo_passwd')
project.project_servers = json.dumps(request.POST.get('project_servers').split(','))
project.project_target_root = request.POST.get('project_target_root')
project.project_logpath = request.POST.get('project_logpath')
project.save()
except Exception as ex:
logger.warn(msg="修改项目部署失败: {ex}".format(ex=ex))
return "修改项目部署失败: {ex}".format(ex=ex)
return project
def create_apps(self,request):
try:
project = Project_Assets.objects.get(id=request.POST.get('project_id'))
except Exception as ex:
logger.warn(msg="查询项目失败: {ex}".format(ex=ex))
try:
apps = Project_Config.objects.create(
project = project,
project_service = request.POST.get('project_service'),
project_type = request.POST.get('project_type'),
project_env = request.POST.get('project_env'),
project_name = request.POST.get('project_name'),
project_is_include = request.POST.get('project_is_include'),
project_repertory = request.POST.get('project_repertory'),
project_address = request.POST.get('project_address'),
project_repo_dir = request.POST.get('project_repo_dir'),
project_remote_command = request.POST.get('project_remote_command'),
project_pre_remote_command = request.POST.get('project_pre_remote_command'),
project_local_command = request.POST.get('project_local_command'),
project_dir = request.POST.get('project_dir'),
project_uuid = uuid.uuid4(),
project_exclude = request.POST.get('project_exclude','.git').rstrip(),
project_user = request.POST.get('project_user','root'),
project_model = request.POST.get('project_model'),
project_status = 0,
project_repo_user = request.POST.get('project_repo_user'),
project_repo_passwd = request.POST.get('project_repo_passwd'),
project_servers = json.dumps(request.POST.get('project_servers').split(',')),
project_target_root = request.POST.get('project_target_root'),
project_logpath = request.POST.get('project_logpath'),
)
except Exception as ex:
logger.warn(msg="添加项目部署失败: {ex}".format(ex=ex))
return "添加项目部署失败: {ex}".format(ex=ex)
return apps
def source(self,assetsList):
sList = []
resource = []
for assets in assetsList:
data = {}
if hasattr(assets,'server_assets'):
try:
sList.append(assets.server_assets.ip)
data["ip"] = assets.server_assets.ip
data["port"] = int(assets.server_assets.port)
data["username"] = assets.server_assets.username
data["sudo_passwd"] = assets.server_assets.sudo_passwd
if assets.server_assets.keyfile == 0:data["password"] = assets.server_assets.passwd
except Exception, ex:
logger.warn(msg="id:{assets}, error:{ex}".format(assets=assets.id,ex=ex))
elif hasattr(assets,'network_assets'):
try:
sList.append(assets.network_assets.ip)
data["ip"] = assets.network_assets.ip
data["port"] = int(assets.network_assets.port)
data["password"] = assets.network_assets.passwd,
data["username"] = assets.network_assets.username
data["sudo_passwd"] = assets.network_assets.sudo_passwd
data["connection"] = 'local'
except Exception, ex:
logger.warn(msg="id:{assets}, error:{ex}".format(assets=assets.id,ex=ex))
def inventory_detail(request, id, format=None):
"""
Retrieve, update or delete a server assets instance.
"""
try:
inventory = Ansible_Inventory.objects.get(id=id)
except Ansible_Inventory.DoesNotExist:
return Response(status=status.HTTP_404_NOT_FOUND)
if request.method == 'GET':
source = {}
for ds in inventory.inventory_group.all():
source[ds.group_name] = {}
hosts = []
for ser in ds.inventory_group_server.all():
assets = Assets.objects.get(id=ser.server)
if hasattr(assets, 'server_assets'):
hosts.append(assets.server_assets.ip)
elif hasattr(assets, 'network_assets'):
hosts.append(assets.network_assets.ip)
source[ds.group_name]['hosts'] = hosts
if ds.ext_vars:
try:
source[ds.group_name]['vars'] = eval(ds.ext_vars)
except Exception, ex:
source[ds.group_name]['vars'] = {}
logger.warn(msg="获取资产组变量失败: {ex}".format(ex=ex))
return JsonResponse({"code": 200, "msg": "success", "data": source})
def asset_info(request, id,format=None):
"""
Retrieve, update or delete a server assets instance.
"""
try:
assets = Assets.objects.get(id=id)
except Assets.DoesNotExist:
return Response(status=status.HTTP_404_NOT_FOUND)
if request.method == 'POST':
dataList = []
try:
if assets.assets_type in ['server','vmser']:
dataList.append({"name":'CPU型号',"value":assets.server_assets.cpu})
dataList.append({"name":'CPU个数',"value":assets.server_assets.vcpu_number})
dataList.append({"name":'硬盘容量',"value":str(int(assets.server_assets.disk_total)/1024)+'GB'})
dataList.append({"name":'内存容量',"value":str(int(assets.server_assets.ram_total)/1024)+'GB'})
dataList.append({"name":'操作系统',"value":assets.server_assets.system})
dataList.append({"name":'内核版本',"value":assets.server_assets.kernel})
dataList.append({"name":'主机名',"value":assets.server_assets.hostname})
else:
dataList.append({"name":'CPU型号',"value":assets.network_assets.cpu})
dataList.append({"name":'内存容量',"value":assets.network_assets.stone})
dataList.append({"name":'背板带宽',"value":assets.network_assets.bandwidth})
dataList.append({"name":'端口总数',"value":assets.network_assets.port_number})
except Exception ,ex:
logger.warn(msg="获取资产信息失败: {ex}".format(ex=ex))
return JsonResponse({"code":200,"msg":"success","data":dataList})
def __init__(self):
# config 的 config
self.USE_COOKIE_POOL = True if global_config.getRaw('config', 'use_cookie_pool') == 'True' else False
self.COOKIE = global_config.getRaw('config', 'Cookie')
self.USER_AGENT = global_config.getRaw('config', 'user-agent')
self.SAVE_MODE = global_config.getRaw('config', 'save_mode')
self.MONGO_PATH = global_config.getRaw('config', 'mongo_path')
self.REQUESTS_TIMES = global_config.getRaw('config', 'requests_times')
self.UUID = global_config.getRaw('config', 'uuid')
self.TCV = global_config.getRaw('config', 'tcv')
# config 的 detail
self.KEYWORD = global_config.getRaw('detail', 'keyword')
self.LOCATION_ID = global_config.getRaw('detail', 'location_id')
self.CHANNEL_ID = global_config.getRaw('detail', 'channel_id')
self.SEARCH_URL = global_config.getRaw('detail', 'search_url')
assert self.SEARCH_URL == '' or self.SEARCH_URL.endswith('p'), 'search_url 没有拼接p'
self.NEED_FIRST = True if global_config.getRaw('detail', 'need_first') == 'True' else False
try:
self.NEED_SEARCH_PAGES = int(global_config.getRaw('detail', 'need_pages'))
except:
logger.error('need_pages 必须为整数')
exit()
# config 的 proxy
self.USE_PROXY = True if global_config.getRaw('proxy', 'use_proxy') == 'True' else False
if self.USE_PROXY:
try:
self.REPEAT_NUMBER = int(global_config.getRaw('proxy', 'repeat_nub'))
except:
logger.error('repeat_nub 必须为整数')
exit()
else:
self.REPEAT_NUMBER = 0
self.HTTP_EXTRACT = True if global_config.getRaw('proxy', 'http_extract') == 'True' else False
self.HTTP_LINK = global_config.getRaw('proxy', 'http_link')
self.KEY_EXTRACT = True if global_config.getRaw('proxy', 'key_extract') == 'True' else False
self.KEY_ID = global_config.getRaw('proxy', 'key_id')
self.KEY_KEY = global_config.getRaw('proxy', 'key_key')
assert not (self.HTTP_EXTRACT is True and self.KEY_EXTRACT is True), '代理模式不可以全为True'
# require 的 shop phone
self.NEED_DETAIL = True if require_config.getRaw('shop_phone', 'need') == 'True' else False
self.NEED_PHONE_DETAIL = True if require_config.getRaw('shop_phone', 'need_detail') == 'True' else False
if self.NEED_PHONE_DETAIL:
logger.warn('开启了电话详情模式，会降低速度并增加反爬概率')
# require 的 shop review
self.NEED_REVIEW = True if require_config.getRaw('shop_review', 'need') == 'True' else False
self.NEED_REVIEW_DETAIL = True if require_config.getRaw('shop_review', 'more_detail') == 'True' else False
if self.NEED_REVIEW_DETAIL:
logger.warn('开启了评论详情模式，会降低速度并增加反爬概率')
try:
self.NEED_REVIEW_PAGES = int(require_config.getRaw('shop_review', 'need_pages'))
except:
logger.error('need_pages 必须为整数')
exit()
else:
self.NEED_REVIEW_PAGES = 0
def ansible_inventory_modf(request,id):
if request.method == "GET":
try:
inventory = Ansible_Inventory.objects.get(id=id)
except Exception,ex:
logger.warn(msg="获取资产组失败: {ex}".format(ex=str(ex)))
serverList = AssetsSource().serverList()
return render(request,'apps/apps_inventory_modf.html',{"user":request.user,"serverList":serverList,"inventory":inventory})
def inventory(self,inventory):
sList = []
resource = {}
groups=''
try:
inventory = Ansible_Inventory.objects.get(id=inventory)
except Exception, ex:
logger.warn(msg="资产组查询失败：{id}".format(id=inventory,ex=ex))
def _load_pinyin_2_word(self, path):
result = dict()
if not os.path.exists(path):
logger.warn("file not exists: %s" % path)
return result
with codecs.open(path, 'r', encoding='utf-8') as f:
a = f.read()
result = eval(a)
return result
def delete(redisKey):
try:
redisConn = APBase.getRedisConnection(APBase.REDSI_POOL)
redisConn.delete(redisKey)
redisConn = None
except Exception as ex:
logger.warn(msg="Delete redis key failed: {ex}".format(
ex=str(ex)))
return False
def article_category(request, pid):
try:
category = Category.objects.get(id=pid)
except Exception, ex:
logger.warn(msg="分类不存在: {ex}".format(ex=ex))
return render(request, 'wiki/wiki_base.html', {
"user": request.user,
"errorInfo": "分类不存在: {ex}".format(ex=ex)
})
def article_tag(request, pid):
try:
tag = Tag.objects.get(id=pid)
except Exception, ex:
logger.warn(msg="标签不存在: {ex}".format(ex=ex))
return render(request, 'wiki/wiki_base.html', {
"user": request.user,
"errorInfo": "标签不存在: {ex}".format(ex=ex)
})
def delete_user_database(self, request):
try:
Database_User.objects.get(db=QueryDict(request.body).get('id'),
user=QueryDict(
request.body).get('uid')).delete()
except Exception as ex:
logger.warn(msg="删除用户数据库失败: {ex}".format(ex=ex))
return "删除用户数据库失败: {ex}".format(ex=ex)
return True
def lpush(redisKey, data):
try:
redisConn = APBase.getRedisConnection(APBase.REDSI_POOL)
redisConn.lpush(redisKey, data)
redisConn = None
except Exception as ex:
logger.warn(msg="Lpush redis data failed: {ex}".format(
ex=str(ex)))
return False
def get_number(self,project):
number_list = []
for num in project.project_number.all():
number = self.convert_to_dict(num)
try:
number['ip'] = self.assets(num.server).server_assets.ip
except Exception as ex:
logger.warn(msg="获取项目成员主机信息失败: {ex}".format(ex=ex))
number_list.append(number)
return number_list
def delete_node(self, request):
node = self.schedNode(request)
if node:
try:
node.delete()
except Exception as ex:
logger.warn(msg="修改节点信息失败: {ex}".format(ex=ex))
return str(ex)
else:
return "节点不存在"
def evaluate(self):
""" Evaluates an agent stored in chekpoint with @self._config.ckpt_num. """
step, update_iter = self._load_ckpt(ckpt_num=self._config.ckpt_num)
logger.info('Run %d evaluations at step=%d, update_iter=%d',
self._config.num_eval, step, update_iter)
for i in trange(self._config.num_eval):
logger.warn("Evalute run %d", i + 1)
rollout, info = self._evaluate(step=step, idx=i)
def set(redisKey, value):
try:
redisConn = APBase.getRedisConnection(APBase.REDSI_POOL)
redisConn.set(redisKey, value)
redisConn.expire(redisKey, 300)
redisConn = None
except Exception as ex:
logger.warn(msg="Set redis key failed: {ex}".format(
ex=str(ex)))
return False
def rpop(redisKey):
try:
redisConn = APBase.getRedisConnection(APBase.REDSI_POOL)
data = redisConn.rpop(redisKey)
redisConn = None
return data
except Exception as ex:
logger.warn(msg="Rpop redis data failed: {ex}".format(
ex=str(ex)))
return False
def get(redisKey):
try:
redisConn = APBase.getRedisConnection(APBase.REDSI_POOL)
result = redisConn.get(redisKey)
redisConn = None
return result
except Exception as ex:
logger.warn(msg="Get redis key failed: {ex}".format(
ex=str(ex)))
return False
def save_to_collect(self, published_time, m_hash, m_count):
save_to_collect = """INSERT INTO `sys_collect_count` ( published_time, hash,count )VALUES('{published_time}' , '{m_hash}','{m_count}' )"""
try:
# 执行sql语句
self.cursor.execute(save_to_collect.format(published_time=published_time, m_hash=m_hash, m_count=m_count))
# 执行sql语句
self.db.commit()
except Exception as e:
# 发生错误时回滚
self.db.rollback()
logger.warn('Mysql Insert Error:save_to_collect' + str(e))
def get_user(self, request):
if request.method == 'GET': cid = request.GET.get('id')
elif request.method == 'POST': cid = request.POST.get('id')
elif request.method in ['PUT', 'DELETE']:
cid = QueryDict(request.body).get('id')
try:
user = User.objects.get(id=cid)
return user
except Exception as ex:
logger.warn(msg="获取用户信息失败: {ex}".format(ex=ex))
return False
def save_to_post(self, title, source_url, s_from, content, published_time, m_hash):
save_to_post = """INSERT INTO `sys_posts` ( `sys_posts`.title, `sys_posts`.source_url, `sys_posts`.from, `sys_posts`.content, `sys_posts`.published_time, `sys_posts`.hash )VALUES( %s,%s,%s,%s,%s,%s )"""
try:
# 执行sql语句
self.cursor.execute(save_to_post,(title, source_url, s_from, content,published_time,m_hash))
# 执行sql语句
self.db.commit()
except Exception as e:
# 发生错误时回滚
self.db.rollback()
logger.warn('Mysql Insert Error:save_to_post' + str(e))
