class NNTPSource(Source):
PREFIX = 'dispose'
def __init__(self, serverName, group, howmany):
super().__init__()
self.serverName = serverName
self.type = Source.NNTP
self.group = group
self.howmany = howmany
self.server = NNTP(self.serverName)
def getNewsItem(self):
resp, count, first, last, name = self.server.group(self.group)
resp = resp.split(' ')[0]
if resp == '211': # 正常响应
start = last - self.howmany + 1
resp, overviews = self.server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = self.server.body(id)
body = '\n'.join(
line.decode('latin')
for line in info.lines) + '\n\n' # 使用生成器推导,转字符串
yield NewsItem(title, body, self.NNTP)
else:
yield None
self.server.quit()
def getDisposeName(self):
return self.PREFIX + self.type
pass
def disposeNNTP(self, news):
print('NNTP新闻生产完毕')
pass
def get_items(self): #新闻生成器
server = NNTP(self.servername)
resp, count, first, last, name = server.group(self.group) #新闻组信息列表
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = server.body(id)
body = '\n'.join(line.decode() for line in info.lines) + '\n\n'
yield NewsItem(title, body)
server.quit()
def get_items(self):
server = NNTP(self.servername)
resp, count, first, last, name = server.group(self.group)
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = server.body(id)
body = '\n'.join(line.decode('latin')
for line in info.lines) + '\n\n'
yield NewsItem(title, body)
server.quit()
def get_items(self):
server = NNTP(self.servername)
resp, count, first, last, name = server.group(self.group)
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over["subject"])
resp, info = server.body(id)
body = "\n".join(line.decode("latin")
for line in info.lines) + "\n\n"
yield NewsItem(title, body)
server.quit()
def get_items(self):
server = NNTP(self.servername)
_, count, first, last, name = server.group(self.group)
start = last - self.how_many + 1
_, overviews = server.over((start, last))
for ID, over in overviews:
title = decode_header(over['subject'])
_, info = server.body(ID)
body = '\n'.join(line.decode('latin') for line in info.lines)
yield NewsItem(title, body)
server.quit()
def get_items(self):
server = NNTP(self.servername)
#服务器响应、新闻组包含的消息数、第一条和最后一条消息编号、新闻组名称
resp, count, first, last, name = server.group(self.group)
#确定要获取的文章编号区间的起始位置
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = server.body(id)
body = '\n'.join(line.decode('latin')
for line in info.lines) + '\n\n'
yield NewsItem(title, body)
server.quit()
def getItems(self):
# yesterday = localtime(time()-self.window*day)
# date = strftime('%y%m%d',yesterday)
# time = strftime('%H%M%S',yesterday)
# create a nntp server
s = NNTP(self.servername)
resp,count,first,last,name = s.group(self.groupname)
resp,overviews = s.over((last-1,last))
for num,over in overviews:
title = over.get('subject')
resp,body = s.body(num)
# create a generator to iterate news
if title and body:
yield NewsItem(title,body)
s.quit()
def get_items(self):
for servername in KNOWN_NNTP_SERVERS:
try:
server = NNTP(servername)
resp, count, first, last, name = server.group(self.group)
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = server.body(id)
body = '\n'.join(
line.decode('latin1') for line in info.lines) + '\n\n'
yield NewsItem(title, body, "NNTP NewsGroup " + self.group)
server.quit()
break
except:
continue
return []
# -*- coding:utf-8 -*-
from nntplib import NNTP
servername = 'web.aioe.org'
group = 'comp.lang.python.announce'
server = NNTP(servername)
howmany = 10
resp, count, first, last, name = server.group(group)
start = last - howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
subject = over['subject']
resp, info = server.body(id)
print('subject:', subject)
print('-' * len(subject))
for line in info.lines:
print(line.decode('latin1'))
print()
server.quit()
('211 98 3911 4008 comp.lang.python.announce', 98, 3911, 4008, 'comp.lang.python.announce')
还有可能得到一个带有类似错误信息的的异常 如果引发了异常 可能是服务器的名字写错了
另外一种可能是在创建服务器对象和调用group方法之间 '超时了'， 服务器允许用户保持连接的时间很短(比如10秒)
如果输入速度不够快 把代码放到脚本里， 或着将服务器对象的创建和方法的调用放在同一行内 (以分号隔开)
'''
'''-----------------------------------------------------------------------------------------------------------'''
# 模块介绍 https://docs.python.org/3/library/nntplib.html
import nntplib
server = nntplib.NNTP('news.gmane.org')
resp, count, frist, last, name = server.group('gmane.comp.python.committers')
print('Group--', name, 'has--', count, 'articles, range--', frist, 'to--', last)
resp, overviews = server.over((last -9, last))
print(resp, '----', overviews)#返回消息头和概述信息
for id , over in overviews:
print(id, nntplib.decode_header(over['subject']))
print(server.body(id)[1])#所有文章信息
print('-----------------', resp)#响应信息
server.quit()
"""
当头decode_header()可以包含非ASCII字符时，建议使用该函数：
NNTP.over（message_spec，*，file = None ）
在旧服务器上发送OVER命令或XOVER命令。 message_spec可以是表示消息id的字符串，也可以是表示当前组中的文章范围的（第一，最后）元组元组，或者指示从第一个到最后一个文章开始的文章范围的（第一，无）元组 当前组中的文章，或“无”选择当前组中的当前文章。
返回一对（响应，概述）。 概述是（article_number，概述）元组的列表，一个用于每个由message_spec选择的文章。 每个概述都是一个具有相同数量项目的字典，但这个数字取决于服务器。
class Client(Context):
def __init__(self,
log,
hostname,
port=None,
username=None,
password=None,
*,
debuglevel=None,
**timeout):
self.log = log
self.hostname = hostname
self.port = port
self.username = username
self.password = password
self.debuglevel = debuglevel
self.timeout = timeout
Context.__init__(self)
self.connect()
def connect(self):
address = net.format_addr((self.hostname, self.port))
self.log.write("Connecting to {}\n".format(address))
if self.port is None:
port = ()
else:
port = (self.port, )
self.connect_time = time.monotonic()
self.nntp = NNTP(self.hostname, *port, **self.timeout)
with ExitStack() as cleanup:
cleanup.push(self)
if self.debuglevel is not None:
self.nntp.set_debuglevel(self.debuglevel)
self.log.write("{}\n".format(self.nntp.getwelcome()))
if self.username is not None:
self.log.write("Logging in as {}\n".format(self.username))
with self.handle_abort():
self.nntp.login(self.username, self.password)
self.log.write("Logged in\n")
cleanup.pop_all()
def body(self, id, *pos, **kw):
id = "<{}>".format(id)
retry = 0
while True:
try:
with self.handle_abort():
self.nntp.body(id, *pos, **kw)
break
except failure_responses as err:
[code, *msg] = err.response.split(maxsplit=1)
if code == "400":
[msg] = msg or (None, )
if not msg:
msg = "Server shut down connection"
elif code[1] == "0" and not retry:
msg = err.response
else:
raise
self.log.write(msg + "\n")
self.log_time()
if retry >= 60:
raise TimeoutError()
self.close()
time.sleep(retry)
if not retry:
start = time.monotonic()
self.connect()
if retry:
retry *= 2
else:
retry = time.monotonic() - start
if retry <= 0:
retry = 0.5
def group(self, *pos, **kw):
with self.handle_abort():
return self.nntp.group(*pos, **kw)
def over(self, *pos, **kw):
with self.handle_abort():
return self.nntp.over(*pos, **kw)
def hdr(self, *pos, **kw):
with self.handle_abort():
return self.nntp.xhdr(*pos, **kw)
@contextmanager
def handle_abort(self):
try:
yield
except failure_responses:
raise # NNTP connection still intact
except:
# Protocol is disrupted so abort the connection straight away
self.close()
raise
def close(self):
if not self.nntp:
return
# Ignore failure of inappropriate QUIT command
with suppress(NNTPError), self.nntp:
pass
self.nntp = None
def log_time(self):
# TODO: time duration formatter
self.log.write("Connection lasted {:.0f}m\n".format(
(time.monotonic() - self.connect_time) / 60))
from nntplib import NNTP
servername = 'news.foo.bar'
group = 'comp.lang.python.announce'
server = NNTP(servername)
howmany = 10
resp, count, first, last, name = server.group(group)
start = last - howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
subject = over['subject']
resp, info = server.body(id)
print(subject)
print('-' * len(subject))
for line in info.lines:
print(line.decode('latin1'))
print()
server.quit()
date = strftime('%y%m%d',yesterday)
time = strftime('%H%M%S',yesterday)
servername = 'news.gmane.org'
# 18289
groupname = 'gmane.comp.python.apple'
s = NNTP(servername)
# group
'Return a tuple (response, count, first, last, name) where count is the (estimated) number of articles in the group, first is the first article number in the group, last is the last article number in the group, and name is the group name. The numbers are returned as strings.'
resp,count,first,last,name = s.group(groupname)
print('Group', name, 'has', count, 'articles, range', first, 'to', last)
# over
'Return a pair (response, overviews). overviews is a list of (article_number, overview) tuples, one for each article selected by message_spec'
resp,overviews = s.over((last-1,last))
for num,over in overviews:
print(num)# 1-100
#print(over)
# print(over.keys())
# ['xref', 'from', ':lines', ':bytes', 'references', 'date', 'message-id', 'subject']
print(over.get('date'))
print(nntplib.decode_header(over.get('from')))
print(over.get('message-id'))
print(over.get('subject'))
# stat
'Return a triple (response, number, id) where number is the article number and id is the message id.'
resp,num,msg_id = s.stat(last)
print(num,msg_id)
class Client(Context):
def __init__(self, log,
hostname, port=None, username=None, password=None, *,
debuglevel=None, **timeout):
self.log = log
self.hostname = hostname
self.port = port
self.username = username
self.password = password
self.debuglevel = debuglevel
self.timeout = timeout
Context.__init__(self)
self.connect()
def connect(self):
address = net.format_addr((self.hostname, self.port))
self.log.write("Connecting to {}\n".format(address))
if self.port is None:
port = ()
else:
port = (self.port,)
self.connect_time = time.monotonic()
self.nntp = NNTP(self.hostname, *port, **self.timeout)
with ExitStack() as cleanup:
cleanup.push(self)
if self.debuglevel is not None:
self.nntp.set_debuglevel(self.debuglevel)
self.log.write("{}\n".format(self.nntp.getwelcome()))
if self.username is not None:
self.log.write("Logging in as {}\n".format(self.username))
with self.handle_abort():
self.nntp.login(self.username, self.password)
self.log.write("Logged in\n")
cleanup.pop_all()
def body(self, id, *pos, **kw):
id = "<{}>".format(id)
retry = 0
while True:
try:
with self.handle_abort():
self.nntp.body(id, *pos, **kw)
break
except failure_responses as err:
[code, *msg] = err.response.split(maxsplit=1)
if code == "400":
[msg] = msg or (None,)
if not msg:
msg = "Server shut down connection"
elif code[1] == "0" and not retry:
msg = err.response
else:
raise
self.log.write(msg + "\n")
self.log_time()
if retry >= 60:
raise TimeoutError()
self.close()
time.sleep(retry)
if not retry:
start = time.monotonic()
self.connect()
if retry:
retry *= 2
else:
retry = time.monotonic() - start
if retry <= 0:
retry = 0.5
def group(self, *pos, **kw):
with self.handle_abort():
return self.nntp.group(*pos, **kw)
def over(self, *pos, **kw):
with self.handle_abort():
return self.nntp.over(*pos, **kw)
def hdr(self, *pos, **kw):
with self.handle_abort():
return self.nntp.xhdr(*pos, **kw)
@contextmanager
def handle_abort(self):
try:
yield
except failure_responses:
raise # NNTP connection still intact
except:
# Protocol is disrupted so abort the connection straight away
self.close()
raise
def close(self):
if not self.nntp:
return
# Ignore failure of inappropriate QUIT command
with suppress(NNTPError), self.nntp:
pass
self.nntp = None
def log_time(self):
# TODO: time duration formatter
self.log.write("Connection lasted {:.0f}m\n".format((time.monotonic() - self.connect_time)/60))
