def test_log_handler():
log = LogHandler("Tlog")
log.info("test log")
log.resetName("test1")
log.info('this is a log from test1')
log.resetName('test2')
log.info('this is a log from test2')
def __init__(self, dbtype='sqlit'):
"""
:param dbtype: 选择数据库类型
"""
self.log = LogHandler("db")
DBCONFIG = DBConfig().get_db_config(dbtype)
ROOT_PATH = os.path.join(os.path.dirname(os.path.abspath(CURRENT_PATH)), DBCONFIG.get('path'))
DB_NAME = DBCONFIG.get("dbname")
DB_PATH = os.path.join(ROOT_PATH,DB_NAME)
print(DB_PATH)
self.conn = sqlite3.connect(DB_PATH)
self.c = self.conn.cursor()
def __init__(self, dbtype):
"""
创建数据库
:param dbtype: 数据库类型
"""
self.log = LogHandler("db")
dbconfig = DBConfig().get_db_config(dbtype)
# self.connection = pymysql.connect(
# **dbconfig,
# )
if dbtype == "mysql":
# print("ok")
self.connection = pymysql.connect(**dbconfig, )
def __init__(self, urltype):
"""
:param urltype: 0-国内高匿代理IP;1-国内透明代理IP;2-国内HTTPS代理IP;3-国外高匿代理IP
"""
url_list = {
0: 'http://www.pcdaili.com/index.php?m=daili&a=free&type=1',
1: 'http://www.pcdaili.com/index.php?m=daili&a=free&type=2',
2: 'http://www.pcdaili.com/index.php?m=daili&a=free&type=3',
3: 'http://www.pcdaili.com/index.php?m=daili&a=free&type=4',
}
if urltype in [0, 1, 2, 3]:
self.url = url_list.get(urltype)
self.ua = UserAgent()
self.sqlite = SqliteClient()
self.sqlite.create_table_sqlite()
self.log = LogHandler("db")
def __init__(self):
"""
构造默认 header request session
"""
self.header = {
"Accept":
"*/*",
"Accept-Encoding":
"gzip,deflate,sdch",
"Accept-Language":
"zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4",
"Connection":
"keep-alive",
"Content-Type":
"application/x-www-form-urlencoded",
"Host":
"music.163.com",
"Referer":
"http://music.163.com",
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"
}
self.session = requests.session()
self.log = LogHandler('NeteaseApi')
def __init__(self, ):
self.config = ConfigParser()
self.name = "config.ini"
self.sql_path = os.path.join(ROOT_PATH, self.name)
self.log = LogHandler("db")
"""
-------------------------------------------
File Name: jinja2html
Description:
Author: wanglin
Date: 2017/12/28
--------------------------------------------
Change Activity:2017/12/28;
--------------------------------------------
"""
__author__ = 'wanglin'
import os
from jinja2 import Environment, FileSystemLoader
from util.LogHandler import LogHandler
log = LogHandler('jinja2html')
PATH = os.path.dirname(os.path.abspath(__file__))
TEMPLATE_ENVIRONMENT = Environment(autoescape=False,
loader=FileSystemLoader(
os.path.join(PATH, 'templates')),
trim_blocks=False)
def create_html(nameinfo, datainfo, tableinfo):
context = {'info': nameinfo, 'datainfo': datainfo, 'tableinfo': tableinfo}
log.info('Jinja1 context info: {}'.format(context))
html = TEMPLATE_ENVIRONMENT.get_template('base.html').render(context)
log.info('Successful rendering report page. ')
return html
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('valid_schedule')
--------------------------------------------
"""
__author__ = 'wanglin'
import os
import json
import cx_Oracle
import MySQLdb
from DBUtils.PooledDB import PooledDB
from util.ConfigHandler import ConfigHandler
from util.Singleton import Singleton
from util.LogHandler import LogHandler
os.environ['NLS_LANG'] = 'SIMPLIFIED CHINESE_CHINA.UTF8'
log = LogHandler('DBManager')
class DBManager(object):
__metaclass__ = Singleton
def __init__(self, option='oracle'):
self.option = option
self.config = ConfigHandler().get(self.option)
if option == 'oracle':
dsn = '{host}:{port}/{db}'.format(host=self.config['host'], port=self.config['port'], db=self.config['db'])
connKwargs = {'user': self.config['user'], 'password': self.config['passwd'], 'dsn': dsn}
self._pool = PooledDB(cx_Oracle, mincached=2, maxcached=2, maxshared=5, maxconnections=10, **connKwargs)
else:
connKwargs = {'host': self.config['host'], 'port': int(self.config['port']), 'user': self.config['user'],
'passwd': self.config['passwd'], 'db': self.config['db'], 'charset': self.config['charset']}
self._pool = PooledDB(MySQLdb, mincached=2, maxcached=2, maxshared=5, maxconnections=10, **connKwargs)
Date : 2019/12/1
-------------------------------------------------
"""
import platform
from flask import Flask, request
from common.Response import Response
from config.Getter import config
from service.MusicooService import MusicooService
from util.LogHandler import LogHandler
app = Flask(__name__)
log = LogHandler('Musicoo')
@app.route('/', methods=['GET'])
def index():
return 'index'
@app.route('/netease/song/<song_id>/url', methods=['GET'])
def song_url(song_id):
"""
获取音乐链接
/netease/song/1379444316/url
:param song_id:
:return:
"""
--------------------------------------------
"""
__author__ = 'wanglin'
import datetime
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
# 引入外部文件
from jinja2html import create_html
from monitorSpider import get_info
from util.LogHandler import LogHandler
from util.DBManager import get_table_count
log = LogHandler('mailsend')
_sender_address = '*****@*****.**'
_reciver_address = '*****@*****.**'
_subject = u'平台报告-【%s】' % datetime.datetime.now().strftime('%Y-%m-%d')
_passwd = 'passwd'
_smtpadd = 'smtp.ruifucredit.com'
def sendMail(sender, reciver, subject, content, passwd, smtpadd):
log.info('Start to initialize the mail message.')
username = sender
password = passwd
msg = MIMEMultipart('related')
msg['Subject'] = subject
# html格式
html = content
File Name: monitorSpider
Description:
Author: wanglin
Date: 2017/12/28
--------------------------------------------
Change Activity:2017/12/28;
--------------------------------------------
"""
__author__ = 'wanglin'
import json
import requests
import datetime
from util.LogHandler import LogHandler
log = LogHandler('monitorSpider')
name = {'Total': '-', 'Used': '-', 'Free': '-', 'PercentUsed': '-', 'TotalBlocks': '-', 'TotalFiles': '-',
'SoftwareVersion': '-'}
node = {'name': '-', 'lastContact': '-', 'xferaddr': '-', 'adminState': '-', 'capacity': '-', 'usedSpace': '-',
'blockPoolUsedPercent': '-', 'version': '-', }
def get_info():
url = 'http://192.168.88.1:50070/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo'
try:
r = requests.get(url=url)
except Exception as ex:
print(ex)
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('refresh_schedule')
def __init__(self):
self.db = DbClient()
self.config = GetConfig()
self.raw_proxy_queue = 'raw_proxy'
self.log = LogHandler('proxy_manager')
self.useful_proxy_queue = 'useful_proxy'
# -*- coding: utf-8 -*-
import requests
from lxml import etree
from util.LogHandler import LogHandler
from util.WebRequest import WebRequest
logger = LogHandler(__name__)
def robustCrawl(func):
def decorate(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
logger.info(u"sorry,主区出错。原因:")
logger.info(e)
return decorate
def verifyProxyFormat(proxy):
import re
verify_regex = r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}'
return True if re.findall(verify_regex, proxy) else False
def getHtmlTree(url, **kwargs):
header = {
'Connection': 'keep-alive',
'Cache-Control': 'max-age=0',
'Upgrade-Insecure-Requests': '1',