class HTTPClient(object):
def __init__(self, url, method='GET', headers=None, cookie=None):
"""
:param method:
:param headers: Must be a dict. Such as headers={'Content_Type':'text/html'}
"""
self.logger = Logger(__name__).get_logger()
self.url = url
self.session = requests.session()
self.method = method.upper()
self.headers = headers
self.cookie = cookie
self._set_header()
self._set_cookie()
def _set_header(self):
"""设置header"""
if self.headers:
self.session.headers.update(self.headers)
self.logger.info('Set headers: {0}'.format(self.headers))
def _set_cookie(self):
"""设置cookie"""
if self.cookie:
self.session.cookies.update(self.cookie)
self.logger.info('Set cookies: {0}'.format(self.cookie))
def _check_method(self):
"""检查传入的method是否可用。"""
if self.method not in METHODS:
self.logger.exception(
UnSupportMethod(u'不支持的method:{0}，请检查传入参数！'.format(
self.method)))
else:
return True
def send(self, params=None, data=None, **kwargs):
"""send request to url.If response 200,return response, else return None."""
if self._check_method():
response = self.session.request(method=self.method,
url=self.url,
params=params,
data=data,
**kwargs)
self.logger.info('{0} {1}.'.format(self.method, self.url))
if response.status_code == 200:
self.logger.info('request success: {0}\n{1}'.format(
response, response.content.strip()))
return response
else:
self.logger.error('request failed: {0} {1}'.format(
response, response.reason))
class IndexDatabase:
def __init__(self):
db = MongoClient(host=os.getenv('MONGO_HOST', config.MONGO['host']),
port=int(
os.getenv('MONGO_PORT',
config.MONGO['port']))).inverted_index
self._constants = db.constants
self._index = db.index
self._userdata = db.userdata
self.__logger = Logger().get_logger(__name__)
def get(self, word, doc=None):
query = self._index.find_one({word: {'$exists': True}})
if query is not None:
return query[word] if doc is None else query[word][doc]
else:
self.__logger.error(f'No such word: {word}')
raise KeyError
def get_constant(self, name):
query = self._constants.find_one({name: {'$exists': True}})
if query is not None:
return query[name]
else:
self.__logger.error(f'No such constant: {name}')
raise KeyError
def get_words_list(self):
return [
list(word.keys())[0] for word in self._index.find({}, {'_id': 0})
]
def write_index(self, index, total_docs_count):
try:
self._constants.drop()
self._index.drop()
self._constants.insert_one({'total_docs_count': total_docs_count})
for word in index:
self._index.insert({word: index[word]}, check_keys=False)
except Exception:
self.__logger.exception('Error writing index to the database.')
def insert_api_keys(self, tokens):
for token in tokens:
self._userdata.update({'api_key': token},
{"$set": {
'api_key': token
}},
upsert=True)
def validate_api_key(self, key):
return self._userdata.find_one({'api_key': key}) is not None
class HTTPClient(object):
def __init__(self, url, method='GET', headers=None, cookie=None):
"""
:param method:
:param headers: Must be a dict. Such as headers={'Content_Type':'text/html'}
"""
self.logger = Logger(__name__).get_logger()
self.url = url
self.session = requests.session()
self.method = method.upper()
self.headers = headers
self.cookie = cookie
self._set_header()
self._set_cookie()
def _set_header(self):
"""设置header"""
if self.headers:
self.session.headers.update(self.headers)
self.logger.info('Set headers: {0}'.format(self.headers))
def _set_cookie(self):
"""设置cookie"""
if self.cookie:
self.session.cookies.update(self.cookie)
self.logger.info('Set cookies: {0}'.format(self.cookie))
def _check_method(self):
"""检查传入的method是否可用。"""
if self.method not in METHODS:
self.logger.exception(UnSupportMethod(u'不支持的method:{0}，请检查传入参数！'.format(self.method)))
else:
return True
def send(self, params=None, data=None, **kwargs):
"""send request to url.If response 200,return response, else return None."""
if self._check_method():
response = self.session.request(method=self.method, url=self.url, params=params, data=data, **kwargs)
self.logger.info('{0} {1}.'.format(self.method, self.url))
if response.status_code == 200:
self.logger.info('request success: {0}\n{1}'.format(response, response.content.strip()))
return response
else:
self.logger.error('request failed: {0} {1}'.format(response, response.reason))
class TCPClient(object):
def __init__(self, domain, port):
self.logger = Logger(__name__).return_logger()
self.domain = domain
self.port = port
self.flag = 0 # 连接后置为1
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def connect(self):
"""连接指定IP、端口"""
if not self.flag:
try:
self._sock.connect((self.domain, self.port))
self.logger.info(
'TcpClient connect to {0}:{1} success.'.format(
self.domain, self.port))
self.flag = 1
except socket.error as e:
self.logger.exception(e)
def send(self, send_string):
"""向服务器端发送send_string，并返回信息，若报错，则返回None"""
self.connect()
if self.flag:
try:
self._sock.send(send_string)
self.logger.info(
'TcpClient send "{0}" to server.'.format(send_string))
except socket.error as e:
self.logger.exception(e)
try:
# 没搞清楚 raw-unicode-escape, unicode-escape 区别
rec = self._sock.recv(10240).decode(
'raw-unicode-escape').encode('utf-8')
self.logger.info(
'TcpClient get "{0}" from server.'.format(rec))
return rec
except socket.error as e:
self.logger.exception(e)
def close(self):
"""关闭连接"""
if self.flag:
self._sock.close()
self.logger.info('TcpClient closed.')
class TCPClient(object):
def __init__(self, domain, port):
self.logger = Logger(__name__).get_logger()
self.domain = domain
self.port = port
self.flag = 0 # 连接后置为1
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def connect(self):
"""连接指定IP、端口"""
if not self.flag:
try:
self._sock.connect((self.domain, self.port))
self.logger.info('TCPClient connect to {0}:{1} success.'.format(self.domain, self.port))
self.flag = 1
except socket.error as e:
self.logger.exception(e)
def send(self, send_string):
"""向服务器端发送send_string，并返回信息，若报错，则返回None"""
self.connect()
if self.flag:
try:
self._sock.send(send_string)
self.logger.info('TCPClient send "{0}" to server.'.format(send_string))
except socket.error as e:
self.logger.exception(e)
try:
# 没搞清楚 raw-unicode-escape, unicode-escape 区别
rec = self._sock.recv(10240).decode('raw-unicode-escape').encode('utf-8')
self.logger.info('TCPClient get "{0}" from server.'.format(rec))
return rec
except socket.error as e:
self.logger.exception(e)
def close(self):
"""关闭连接"""
if self.flag:
self._sock.close()
self.logger.info('TCPClient closed.')
class Email:
"""Email类，读取配置中的基本配置，初始化要求title必填，message与文件路径可选"""
def __init__(self,
title,
message=None,
path=None,
server=None,
sender=None,
password=None,
receiver=None):
"""初始化Email
:param title: 邮件标题，必填。
:param message: 邮件正文，非必填。
:param path: 附件路径，可传入list（多附件）或str（单个附件），非必填。
:param server: smtp服务器，如果为空，则读取config.ini中的[email]/server，非必填。
:param sender: 发件人，如果为空，则读取config.ini中的[email]/from，非必填。
:param password: 发件人密码，如果为空，则读取config.ini中的[email]/password，如果读取失败，则需手动输入密码，非必填。
:param receiver: 收件人，如果为空，则读取config.ini中的[email]/to，多收件人用“；”隔开，非必填。
"""
self.logger = Logger(__name__).get_logger()
self.title = title
self.message = message
self.files = path
self.msg = MIMEMultipart('related')
cf = DefaultConfig()
if server:
self.server = server
else:
self.server = cf.get('email', 'server')
if sender:
self.sender = sender
else:
self.sender = cf.get('email', 'from')
if receiver:
self.receiver = receiver
else:
self.receiver = cf.get('email', 'to')
if password:
self.password = password
else:
try:
self.password = cf.get('email', 'pass')
except NoOptionError:
self.password = getpass(
prompt=u'未在config.ini中检测到password，请输入password： ')
def _attach_file(self, att_file):
"""内部方法，将单个文件添加到附件列表中"""
att = MIMEText(open('%s' % att_file, 'rb').read(), 'plain', 'utf-8')
att["Content-Type"] = 'application/octet-stream'
filename = re.split(r'[\\|/]', att_file)
att["Content-Disposition"] = 'attachment; filename="%s"' % filename[-1]
self.msg.attach(att)
self.logger.info('attach file {}'.format(att_file))
def send(self):
"""组织邮件内容并发送邮件。"""
self.msg['Subject'] = self.title
self.msg['From'] = self.sender
self.msg['To'] = self.receiver
# 邮件正文
if self.message:
self.msg.attach(MIMEText(self.message))
# 添加附件，支持多个附件（传入list），或者单个附件（传入str）
if self.files:
if isinstance(self.files, list):
for f in self.files:
self._attach_file(f)
elif isinstance(self.files, str):
self._attach_file(self.files)
# 连接服务器并发送
try:
smtp_server = smtplib.SMTP(self.server)
except (gaierror and error) as e:
self.logger.exception(u'发送邮件失败,无法连接到SMTP服务器，检查网络以及SMTP服务器. %s', e)
else:
try:
smtp_server.login(self.sender, self.password)
except smtplib.SMTPAuthenticationError as e:
self.logger.exception(u'用户名密码验证失败！%s', e)
else:
smtp_server.sendmail(self.sender, self.receiver.split(';'),
self.msg.as_string())
finally:
smtp_server.quit()
self.logger.info(u'发送邮件"{0}"成功! 收件人：{1}。如果没有收到邮件，请检查垃圾箱，'
u'同时检查收件人地址是否正确'.format(
self.title, self.receiver))
class Email:
"""Email类，读取配置中的基本配置，初始化要求title必填，message与文件路径可选"""
def __init__(self, title, message=None, path=None, server=None, sender=None, password=None, receiver=None):
"""初始化Email
:param title: 邮件标题，必填。
:param message: 邮件正文，非必填。
:param path: 附件路径，可传入list（多附件）或str（单个附件），非必填。
:param server: smtp服务器，如果为空，则读取config.ini中的[email]/server，非必填。
:param sender: 发件人，如果为空，则读取config.ini中的[email]/from，非必填。
:param password: 发件人密码，如果为空，则读取config.ini中的[email]/password，如果读取失败，则需手动输入密码，非必填。
:param receiver: 收件人，如果为空，则读取config.ini中的[email]/to，多收件人用“；”隔开，非必填。
"""
self.logger = Logger(__name__).get_logger()
self.title = title
self.message = message
self.files = path
self.msg = MIMEMultipart('related')
cf = DefaultConfig()
if server:
self.server = server
else:
self.server = cf.get('email', 'server')
if sender:
self.sender = sender
else:
self.sender = cf.get('email', 'from')
if receiver:
self.receiver = receiver
else:
self.receiver = cf.get('email', 'to')
if password:
self.password = password
else:
try:
self.password = cf.get('email', 'pass')
except NoOptionError:
self.password = getpass(prompt=u'未在config.ini中检测到password，请输入password： ')
def _attach_file(self, att_file):
"""内部方法，将单个文件添加到附件列表中"""
att = MIMEText(open('%s' % att_file, 'rb').read(), 'plain', 'utf-8')
att["Content-Type"] = 'application/octet-stream'
filename = re.split(r'[\\|/]', att_file)
att["Content-Disposition"] = 'attachment; filename="%s"' % filename[-1]
self.msg.attach(att)
self.logger.info('attach file {}'.format(att_file))
def send(self):
"""组织邮件内容并发送邮件。"""
self.msg['Subject'] = self.title
self.msg['From'] = self.sender
self.msg['To'] = self.receiver
# 邮件正文
if self.message:
self.msg.attach(MIMEText(self.message))
# 添加附件，支持多个附件（传入list），或者单个附件（传入str）
if self.files:
if isinstance(self.files, list):
for f in self.files:
self._attach_file(f)
elif isinstance(self.files, str):
self._attach_file(self.files)
# 连接服务器并发送
try:
smtp_server = smtplib.SMTP(self.server)
except (gaierror and error) as e:
self.logger.exception(u'发送邮件失败,无法连接到SMTP服务器，检查网络以及SMTP服务器. %s', e)
else:
try:
smtp_server.login(self.sender, self.password)
except smtplib.SMTPAuthenticationError as e:
self.logger.exception(u'用户名密码验证失败！%s', e)
else:
smtp_server.sendmail(self.sender, self.receiver.split(';'), self.msg.as_string())
finally:
smtp_server.quit()
self.logger.info(u'发送邮件"{0}"成功! 收件人：{1}。如果没有收到邮件，请检查垃圾箱，'
u'同时检查收件人地址是否正确'.format(self.title, self.receiver))
