def gmane(self):
try:
gmane = NNTP("news.gmane.org")
except:
return self.gmane
gmane.group(self.group)
return gmane
def gmane(self):
try:
gmane = NNTP("news.gmane.org")
except:
return self.gmane
gmane.group(self.group)
return gmane
def getItems(self):
server = NNTP(self.servername)
group = server.group(self.group)
first = group[2]
last = str(int(group[2]) + 10)
#last = str(int(group[2]) + 2)
for id in range(int(first), int(last)):
lines = server.article(str(id))[3]
message = message_from_string('\n'.join(lines))
title = message['subject']
#print(repr(dir(message)))
print(message.get_content_type())
if 'text' in message.get_content_type():
body = message.get_payload()
else:
body = 'Body is not text'
if 'ybegin' in message.get_payload():
body = 'hmm'
if message.is_multipart():
#body = body[0]
body = 'Multi part is not implemented yet'
yield NewsItem(title, body)
server.quit()
class NNTPSource(Source):
PREFIX = 'dispose'
def __init__(self, serverName, group, howmany):
super().__init__()
self.serverName = serverName
self.type = Source.NNTP
self.group = group
self.howmany = howmany
self.server = NNTP(self.serverName)
def getNewsItem(self):
resp, count, first, last, name = self.server.group(self.group)
resp = resp.split(' ')[0]
if resp == '211': # 正常响应
start = last - self.howmany + 1
resp, overviews = self.server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = self.server.body(id)
body = '\n'.join(
line.decode('latin')
for line in info.lines) + '\n\n' # 使用生成器推导,转字符串
yield NewsItem(title, body, self.NNTP)
else:
yield None
self.server.quit()
def getDisposeName(self):
return self.PREFIX + self.type
pass
def disposeNNTP(self, news):
print('NNTP新闻生产完毕')
pass
def main():
s = NNTP(settings.nntp_hostname, settings.nntp_port)
resp, groups = s.list()
# check all of the groups, just in case
for group_name, last, first, flag in groups:
resp, count, first, last, name = s.group(group_name)
print "\nGroup", group_name, 'has', count, 'articles, range', first, 'to', last
resp, subs = s.xhdr('subject', first + '-' + last)
for id, sub in subs[-10:]:
print id, sub
s.quit()
def get_items(self): #新闻生成器
server = NNTP(self.servername)
resp, count, first, last, name = server.group(self.group) #新闻组信息列表
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = server.body(id)
body = '\n'.join(line.decode() for line in info.lines) + '\n\n'
yield NewsItem(title, body)
server.quit()
def main():
s = NNTP(settings.nntp_hostname, settings.nntp_port)
resp, groups = s.list()
# check all of the groups, just in case
for group_name, last, first, flag in groups:
resp, count, first, last, name = s.group(group_name)
print "\nGroup", group_name, 'has', count, 'articles, range', first, 'to', last
resp, subs = s.xhdr('subject', first + '-' + last)
for id, sub in subs[-10:]:
print id, sub
s.quit()
def use_NNTP():
for server in NNTP_server_list:
print server
server = raw_input('Which NNTP_server do you want to go?\n')
while True:
if server not in NNTP_server_list:
print 'invalid srever name,try again!'
server = raw_input()
else:
break
s = NNTP(server)
print "s = NNTP('news.newsfan.net') is OK!"
(resp, lst) = s.list()
while True:
for i, elem in enumerate(lst):
print i, elem[0].decode('gbk')
num = raw_input('Which group do you want to go?\n')
try:
rsp, ct, first, last, grp = s.group(lst[int(num)][0])
print "Article's range is:%s to %s." % (first, last)
(resp, subs) = s.xhdr('subject', (str(first) + '-' + str(last)))
except:
print format_exc()
print 'invalid input!try again!!!'
sleep(3)
continue
for subject in subs:
try:
print subject[0], subject[1].decode('gbk')
except:
print subject[0], subject[1]
while True:
try:
number = raw_input('Which article do you want to read?\n')
if number == 'q':
break
f = open("NNTPfile", 'w')
(reply, num, id, list) = s.body(str(number), f)
f = open("NNTPfile", 'r')
for eachLine in f:
try:
print eachLine.decode('gbk'),
except:
print eachLine
f.close()
print 'Press any to continue...(Press q to return...)'
if raw_input() == 'q':
break
except:
print format_exc()
print 'invalid input!try again!!!'
sleep(3)
s.quit()
return
def get_items(self):
server = NNTP(self.servername)
resp, count, first, last, name = server.group(self.group)
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over["subject"])
resp, info = server.body(id)
body = "\n".join(line.decode("latin")
for line in info.lines) + "\n\n"
yield NewsItem(title, body)
server.quit()
def get_items(self):
server = NNTP(self.servername)
resp, count, first, last, name = server.group(self.group)
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = server.body(id)
body = '\n'.join(line.decode('latin')
for line in info.lines) + '\n\n'
yield NewsItem(title, body)
server.quit()
def get_items(self):
server = NNTP(self.servername)
_, count, first, last, name = server.group(self.group)
start = last - self.how_many + 1
_, overviews = server.over((start, last))
for ID, over in overviews:
title = decode_header(over['subject'])
_, info = server.body(ID)
body = '\n'.join(line.decode('latin') for line in info.lines)
yield NewsItem(title, body)
server.quit()
def download_group(name):
if not os.path.exists(name):
os.mkdir(name)
s = NNTP('news.gmane.org')
resp, count, first, last, name = s.group(name)
print 'Group', name, 'has', count, 'articles, range', first, 'to', last
resp, subs = s.xhdr('subject', first + '-' + last)
for id, sub in subs:
print id
with open(os.path.join(name, str(id)), 'wb') as fp:
pprint.pprint(s.article(id), stream=fp)
def get_items(self):
server = NNTP(self.servername)
#服务器响应、新闻组包含的消息数、第一条和最后一条消息编号、新闻组名称
resp, count, first, last, name = server.group(self.group)
#确定要获取的文章编号区间的起始位置
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = server.body(id)
body = '\n'.join(line.decode('latin')
for line in info.lines) + '\n\n'
yield NewsItem(title, body)
server.quit()
def getItems(self):
# yesterday = localtime(time()-self.window*day)
# date = strftime('%y%m%d',yesterday)
# time = strftime('%H%M%S',yesterday)
# create a nntp server
s = NNTP(self.servername)
resp,count,first,last,name = s.group(self.groupname)
resp,overviews = s.over((last-1,last))
for num,over in overviews:
title = over.get('subject')
resp,body = s.body(num)
# create a generator to iterate news
if title and body:
yield NewsItem(title,body)
s.quit()
def getItems(self):
server = NNTP(self.servername)
(resp, count, first, last, name) = server.group(self.group)
(resp, subs) = server.xhdr('subject', (str(first) + '-' + str(last)))
for subject in subs[-10:]:
title = subject[1]
(reply, (num, id, list)) = server.body(subject[0])
# list是一个列表，但是是bytes编码的，需要把每一个元素都解码成string
body = []
#print(list)
for l in list:
body.append(l.decode('gbk')) # 注意，这里用utf-8解码会出现解码错误
#print(body)
body = ''.join(body)
yield NewsItem(title, body)
server.quit()
def getItems(self):
server = NNTP(self.servername)
(resp, count, frist, last, name) = server.group(self.group)
(resp, subs) = server.xhdr('subject', (str(frist) + '-' + (last)))
for subject in subs[-10:]:
title = subject[1]
(reply, num, id, list) = server.body(subject[0])
body = ''.join(list)
#print(num) #186919
#print(title) #Re: Find out which module a class came from
#print(''.join(list))#prano wrote:> But for merely ordinary obfuscation caused by poor...
yield NewsItem(title, body)
server.quit()
def get_items(self):
for servername in KNOWN_NNTP_SERVERS:
try:
server = NNTP(servername)
resp, count, first, last, name = server.group(self.group)
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = server.body(id)
body = '\n'.join(
line.decode('latin1') for line in info.lines) + '\n\n'
yield NewsItem(title, body, "NNTP NewsGroup " + self.group)
server.quit()
break
except:
continue
return []
def getItems(self):
server = NNTP(self.servername)
_, count, first, last, name = server.group(self.group)
_, subs = server.xhdr(
'subject', (str(first) + '-' + str(last)))
for sub in subs[:10]: # default last ten
id = sub[0]
lines = server.article(id)[3]
message = message_from_string('\n'.join(lines))
title = message['subject']
body = message.get_payload()
if message.is_multipart():
body = body[0]
yield NewsItem(title, body)
server.quit()
class NNTPConnector(BaseConnector):
@logit(log,'fetch')
def fetch(self):
"""
Fetches all the messages for a given news group uri and return Fetched staus depending
on the success and faliure of the task
"""
try:
#eg. self.currenturi = nntp://msnews.microsoft.com/microsoft.public.exchange.setup
#nntp_server = 'msnews.microsoft.com'
#nntp_group = 'microsoft.public.exchange.setup'
self.genre = 'review'
try:
nntp_server = urlparse(self.currenturi)[1]
except:
log.exception(self.log_msg("Exception occured while connecting to NNTP server %s"%self.currenturi))
return False
nntp_group = urlparse(self.currenturi)[2][1:]
self.server = NNTP(nntp_server)
try:
self.__updateParentSessionInfo()
resp, count, first, last, name = self.server.group(nntp_group)
last_id = int(last)
first_id = self.__getMaxCrawledId(last_id)+1
log.debug("first_id is %d:"%first_id)
log.debug("last_id is %d:"%last_id)
if last_id >= first_id:
resp, items = self.server.xover(str(first_id), str(last_id))
log.debug(self.log_msg("length of items:%s"%str(len(items))))
for self.id, self.subject, self.author, self.date, self.message_id,\
self.references, size, lines in items:
self.__getMessages(self.task.instance_data['uri'])
self.server.quit()
return True
except:
log.exception(self.log_msg("Exception occured in fetch()"))
self.server.quit()
return False
except Exception,e:
log.exception(self.log_msg("Exception occured in fetch()"))
return False
def getItems(self):
"""
书中原例getItems()方法
返回 nntplib.NNTPTemporaryError: 480 NEWNEWS command disabled by administrator
#480管理员禁用NEWNEWS命令
def getItems(self):
start = localtime(time() - self.window*day)
date = strftime('%y%m%d', start)
hour = strftime('%H%M%S', start)
server = NNTP(self.servername)
ids = server.newnews(self.group, date, hour)[1]
for id in ids:
lines = serverarticle(id)[3]
message = message_from_string('\n'.join(lines))
title = message['subject']
body = message.get_payload()
if message.is_multipat():
body = body[0]
yield NewsItem(title, body)
server.quit()
"""
server = NNTP(self.servername)
(resp, count, frist, last, name) = server.group(self.group)
(resp, subs) = server.xhdr('subject', (str(frist) + '-' + (last)))
for subject in subs[-10:]:
title = subject[1]
(reply, num, id, list) = server.body(subject[0])
body = ''.join(list)
#print(num) #186919
#print(title) #Re: Find out which module a class came from
#print(''.join(list))#prano wrote:> But for merely ordinary obfuscation caused by poor...
yield NewsItem(title, body)
server.quit()
class NewsGrep:
def __init__(self,server,username,password):
self.server = NNTP(server, 119,username,password)
def __del__(self):
pass
def __str__(self):
pass
def list(self):
resp, groups = self.server.list()
for group, last, first, flag in groups:
print group
def ls(self,group_name):
resp, count, first, last, name = self.server.group(group_name)
print 'Group', name, 'has', count, 'articles, range', first, 'to', last
resp, subs = self.server.xhdr('subject', first + '-' + last)
for id, sub in subs[-10:]:
print id, sub
class NewsGrep:
def __init__(self, server, username, password):
self.server = NNTP(server, 119, username, password)
def __del__(self):
pass
def __str__(self):
pass
def list(self):
resp, groups = self.server.list()
for group, last, first, flag in groups:
print group
def ls(self, group_name):
resp, count, first, last, name = self.server.group(group_name)
print 'Group', name, 'has', count, 'articles, range', first, 'to', last
resp, subs = self.server.xhdr('subject', first + '-' + last)
for id, sub in subs[-10:]:
print id, sub
class Client(Context):
def __init__(self, log,
hostname, port=None, username=None, password=None, *,
debuglevel=None, **timeout):
self.log = log
self.hostname = hostname
self.port = port
self.username = username
self.password = password
self.debuglevel = debuglevel
self.timeout = timeout
Context.__init__(self)
self.connect()
def connect(self):
address = net.format_addr((self.hostname, self.port))
self.log.write("Connecting to {}\n".format(address))
if self.port is None:
port = ()
else:
port = (self.port,)
self.connect_time = time.monotonic()
self.nntp = NNTP(self.hostname, *port, **self.timeout)
with ExitStack() as cleanup:
cleanup.push(self)
if self.debuglevel is not None:
self.nntp.set_debuglevel(self.debuglevel)
self.log.write("{}\n".format(self.nntp.getwelcome()))
if self.username is not None:
self.log.write("Logging in as {}\n".format(self.username))
with self.handle_abort():
self.nntp.login(self.username, self.password)
self.log.write("Logged in\n")
cleanup.pop_all()
def body(self, id, *pos, **kw):
id = "<{}>".format(id)
retry = 0
while True:
try:
with self.handle_abort():
self.nntp.body(id, *pos, **kw)
break
except failure_responses as err:
[code, *msg] = err.response.split(maxsplit=1)
if code == "400":
[msg] = msg or (None,)
if not msg:
msg = "Server shut down connection"
elif code[1] == "0" and not retry:
msg = err.response
else:
raise
self.log.write(msg + "\n")
self.log_time()
if retry >= 60:
raise TimeoutError()
self.close()
time.sleep(retry)
if not retry:
start = time.monotonic()
self.connect()
if retry:
retry *= 2
else:
retry = time.monotonic() - start
if retry <= 0:
retry = 0.5
def group(self, *pos, **kw):
with self.handle_abort():
return self.nntp.group(*pos, **kw)
def over(self, *pos, **kw):
with self.handle_abort():
return self.nntp.over(*pos, **kw)
def hdr(self, *pos, **kw):
with self.handle_abort():
return self.nntp.xhdr(*pos, **kw)
@contextmanager
def handle_abort(self):
try:
yield
except failure_responses:
raise # NNTP connection still intact
except:
# Protocol is disrupted so abort the connection straight away
self.close()
raise
def close(self):
if not self.nntp:
return
# Ignore failure of inappropriate QUIT command
with suppress(NNTPError), self.nntp:
pass
self.nntp = None
def log_time(self):
# TODO: time duration formatter
self.log.write("Connection lasted {:.0f}m\n".format((time.monotonic() - self.connect_time)/60))
#!/usr/bin/env python
# -*- coding: UTF-8 *-*
from nntplib import NNTP
n = NNTP('your.nntp.server')
rsp, ct, fst, lst, grp = n.group('comp.lang.python')
rsp, anum, mid, data = n.article('110457')
for eachLine in data:
print eachLine
From: "Alex Martelli" <alex@...> Subject: Re: Rounding Question
Date: Wed, 21 Feb 2001 17:05:36 +0100
"Remco Gerlich" <remco@...> wrote:
Jacob Kaplan-Moss <jacob@...> wrote in comp.lang.python:
So I've got a number between 40 and 130 that I want to round up to
the nearest 10. That is:
40 --> 40, 41 --> 50, ..., 49 --> 50, 50 --> 50, 51 --> 60
Rounding like this is the same as adding 5 to the number and then
rounding down. Rounding down is substracting the remainder if you were
to divide by 10, for which we use the % operator in Python.
This will work if you use +9 in each case rather than +5 (note that he doesn't
really want rounding -- he wants 41 to 'round' to 50, for ex).
Alex
>>> n.quit()
'205 closing connection - goodbye!'
from nntplib import NNTP
from time import strftime,time,localtime
day = 24*60*60
window = 7
yesterday = localtime(time()-window*day)
date = strftime('%y%m%d',yesterday)
time = strftime('%H%M%S',yesterday)
servername = 'news.gmane.org'
# 18289
groupname = 'gmane.comp.python.apple'
s = NNTP(servername)
# group
'Return a tuple (response, count, first, last, name) where count is the (estimated) number of articles in the group, first is the first article number in the group, last is the last article number in the group, and name is the group name. The numbers are returned as strings.'
resp,count,first,last,name = s.group(groupname)
print('Group', name, 'has', count, 'articles, range', first, 'to', last)
# over
'Return a pair (response, overviews). overviews is a list of (article_number, overview) tuples, one for each article selected by message_spec'
resp,overviews = s.over((last-1,last))
for num,over in overviews:
print(num)# 1-100
#print(over)
# print(over.keys())
# ['xref', 'from', ':lines', ':bytes', 'references', 'date', 'message-id', 'subject']
print(over.get('date'))
print(nntplib.decode_header(over.get('from')))
print(over.get('message-id'))
print(over.get('subject'))
def newsgroup(G='', F='', C='', A=None, P=None, RESPONSE=None):
"""The article list for group G in framestyle F"""
if G=='': return "Missing newsgroup name."
group = G
showframes = 1
show_articles = default_show_articles
if os.environ.has_key('HTTP_USER_AGENT'):
browser = os.environ['HTTP_USER_AGENT']
else:
browser = "unknown"
if string.find(browser, "Mozilla/") == 0:
browser_version = string.atof(browser[8:string.index(browser, ' ')])
if browser_version >= 2.00:
showframes = 3
if F != '':
try:
showframes = string.atoi(F)
except AttributeError:
showframes = 0
if C != '':
try:
show_articles = string.atoi(C)
except AttributeError:
show_articles = default_show_articles
user = A
acc_string = ''
if A: acc_string = '&A=' + A
password = P
pass_string = ''
if P: pass_string = '&P=' + P
lines = []
RESPONSE.headers['expires'] = time.asctime(time.gmtime(time.time() + 60))
RESPONSE.write( """<HTML><HEAD><TITLE>Tokyo PC Users Group: %s</TITLE></HEAD>""" % group)
try:
try:
news = NNTP(NEWS_SERVER)
except:
RESPONSE.write( "<BODY><B>Can not connect to server:</B> ", NEWS_SERVER)
raise NewsError
try:
resp = news.shortcmd('MODE READER')
except:
RESPONSE.write( "<BODY><B>Can not communicate with server:</B> ", NEWS_SERVER)
raise NewsError
if user:
resp = news.shortcmd('authinfo user '+user)
if resp[:3] == '381':
if not password:
RESPONSE.write( "<BODY><B>Can not fetch newsgroup</B>")
raise NewsError
else:
resp = news.shortcmd('authinfo pass '+password)
if resp[:3] != '281':
RESPONSE.write( "<BODY><B>Can not fetch newsgroup</B>")
raise NewsError
try:
resp, count, first, last, name = news.group(group)
except:
RESPONSE.write( "<BODY><B>No such newsgroup:</B> " + group )
raise NewsError
description = ""
try:
resp, lines = news.xgtitle(group)
except:
pass
else:
for line in lines:
name, description = line
if showframes == 0:
RESPONSE.write( '<BODY BGCOLOR="#FFFFFF"><H1>%s</H1>' % group)
RESPONSE.write( "<EM>%s</EM><P>" % cgi.escape(description))
elif showframes == 1 or showframes == 3:
if description: description = "&D="+quote(description)
RESPONSE.write( '<FRAMESET ROWS="33%,*">')
RESPONSE.write( ' <FRAMESET COLS="220,*">')
RESPONSE.write( ' <FRAME SRC="/cgi-bin/webnews/logo?G=%s%s" scrolling="auto">' % (group, description))
RESPONSE.write( ' <FRAME SRC="/cgi-bin/webnews/newsgroup?G=%s&F=2%s%s#last" scrolling="yes"> ' % (group, acc_string, pass_string))
RESPONSE.write( ' </FRAMESET>')
if string.find(G, "ttalk") >= 0:
RESPONSE.write( ' <FRAME SRC="http://ttalk.soholutions.com/welcome.html" scrolling="auto" name="d">')
else:
RESPONSE.write( ' <FRAME SRC="/webnews/welcome.html" scrolling="auto" name="d">')
RESPONSE.write( '</FRAMESET><BODY BGCOLOR="#FFFFFF">')
else:
RESPONSE.write( '<BODY BGCOLOR="#FFFFFF">')
if showframes == 3:
raise NewsError
if (show_articles > 0):
ilast = string.atoi(last)
ifirst = string.atoi(first)
if ((ilast - ifirst + 1) > show_articles):
first = "%d" % (ilast - show_articles + 1)
RESPONSE.write( '<A HREF="/cgi-bin/webnews/newsgroup?G=%s&F=%d&C=0%s%s"><I>Retrieve earlier article headers</I></A> ' % (group, showframes, acc_string, pass_string))
try:
resp, lines = news.xover(first, last)
except:
RESPONSE.write( "<B>Unable to get article list for:</B> " + group)
raise NewsError
RESPONSE.write( '<UL TYPE="none">')
# pass 1: build a dictionary of message IDs
ids = {}
index = 0
for line in lines:
art_nr, subject, poster, date, id, references, size, line_cnt = line
ids[id] = index
index = index + 1
# pass 2: discover child articles
childof = []
subs = {}
# subject_re_less = regex.symcomp("\([Rr]e:\)? *\(<real_subject>.*\)")
subject_re_less = re.compile(r"(re:)?\s*(?P<real_subject>.*)")
index = 0
for line in lines:
art_nr, subject, poster, date, id, references, size, line_cnt = line
childof.append(-1)
# if subject_re_less.match(subject) > 0:
# subject = subject_re_less.group('real_subject')
srl = subject_re_less.match(subject)
if srl: subject = srl.group('real_subject')
# if there are references, use them (most recent first)
if len(references) > 0:
references.reverse()
for ref in references:
if ids.has_key(ref):
childof[index] = ids[ref]
break
# if no references (or referee not found), use subject
if childof[index] == -1:
if subs.has_key(subject) :
childof[index] = subs[subject]
else:
subs[subject] = index
index = index + 1
# index = 0
# for line in lines:
# art_nr, subject, poster, date, id, size, line_cnt, references = line
# print index,childof[index],subject
# index = index + 1
index = 0
for seq in childof:
if seq == -1:
show_article_and_kids(index, 0, lines, childof,
acc_string, pass_string, RESPONSE)
index = index + 1
# art_nr, subject, poster, date, id, size, line_cnt, references = line
# name, email = parseaddr(poster)
# print '<LI><A HREF="http:/cgi-bin/readnews.cgi?%s" TARGET="d">%s</A> (%s) %s, %s' % (quote(id), subject, line_cnt, name, time.strftime('%b %d, %H:%M', parsedate(date)))
RESPONSE.write('<A NAME="last"> </A></UL>')
finally:
if showframes != 2:
if string.find(G, "ttalk") >= 0:
RESPONSE.write( """<P><HR><P>A service of the
<A HREF="http://www.soholutions.com/">SoHolutions</A>.""")
else:
RESPONSE.write( """<P><HR><P>A service of the
<A HREF="http://www.tpc.ml.org/">Tokyo PC Users Group</A>.""")
# print "<P><ADDRESS>",os.environ['HTTP_USER_AGENT'],"</ADDRESS>"
RESPONSE.write( """</BODY></HTML>""")
#__author: ZhengNengjin
#__date: 2018/10/25
from nntplib import NNTP
n = NNTP('your.nntp.server')
rsp, ct, fst, lst, grp = n.group('comp,lang.python')
rsp, anum, mid, data = n.article('110457')
for eachLine in data:
print(eachLine)
n.quit()
listonly = 0
showhdrs = ['From', 'Subject', 'Date', 'Newsgroups', 'Lines']
try:
import sys
servername, groupname, showcount = sys.argv[1:]
showcount = int(showcount)
except:
servername = 'news.rmi.net'
groupname = 'comp.lang.python' # cmd line args or defaults
showcount = 10 # show last showcount posts
# connect to nntp server
print 'Connecting to', servername, 'for', groupname
from nntplib import NNTP
connection = NNTP(servername)
(reply, count, first, last, name) = connection.group(groupname)
print '%s has %s articles: %s-%s' % (name, count, first, last)
# get request headers only
fetchfrom = str(int(last) - (showcount-1))
(reply, subjects) = connection.xhdr('subject', (fetchfrom + '-' + last))
# show headers, get message hdr+body
for (id, subj) in subjects: # [-showcount:] if fetch all hdrs
print 'Article %s [%s]' % (id, subj)
if not listonly and raw_input('=> Display?') in ['y', 'Y']:
reply, num, tid, list = connection.head(id)
for line in list:
for prefix in showhdrs:
if line[:len(prefix)] == prefix:
print line[:80]; break
# -*-coding:gbk-*-
from nntplib import NNTP
server = NNTP('news.foo.bar')
print server.group('comp.lang.python.announce')[1]
from nntplib import NNTP
server = NNTP('news.newsfan.net')
server.group('comp.lang.python')[0]
# -*- coding:utf-8 -*-
from nntplib import NNTP
servername = 'web.aioe.org'
group = 'comp.lang.python.announce'
server = NNTP(servername)
howmany = 10
resp, count, first, last, name = server.group(group)
start = last - howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
subject = over['subject']
resp, info = server.body(id)
print('subject:', subject)
print('-' * len(subject))
for line in info.lines:
print(line.decode('latin1'))
print()
server.quit()
class Archive(object):
@staticmethod
def is_diff(body):
return bool([line for line in body if line.startswith("diff ")])
def __init__(self, group, server):
self.conn = NNTP(server)
resp, count, first, last, name = self.conn.group(group)
self.group = group
self.server = server
self.first = int(first)
self.last = int(last)
def get_number_from_user(self, msg_id):
"""
Convert something the user might input into a message id.
These are:
# An NNTP message number
# A gmane link that includes the NNTP message number
# The original Message-Id header of the message.
NOTE: gmane's doesn't include the message number in STAT requests
that involve only the Message-Id (hence the convolution of getting
all the headers).
"""
msg_id = re.sub(r".*gmane.org/gmane.comp.version-control.git/([0-9]+).*", r"\1", str(msg_id))
_, n, id, result = self.conn.head(msg_id)
for header in result:
m = re.match(r"Xref: .*:([0-9]+)\s*$", header, re.I)
if m:
return int(m.group(1))
else:
raise FatalError("No (or bad) Xref header for message '%s'" % msg_id)
def get_patch_series(self, user_input, search_limit=100):
"""
Given an NNTP message number or a Message-Id header return
an mbox containing the patches introduced by the author of that message.
This handles the case where the threading is right *and* the patches
are numbered in a simple scheme:
[PATCH] this patch has no replies and stands on its own
[PATCH 0/2] this is an introduction to the series
|- [PATCH 1/2] the first commit
|- [PATCH 2/2] the second commit
[PATCH 1/3] this is the first commit
|- [PATCH 2/3] and this is the second
|- [PATCH 3/3] and this is the third
TODO: it would be nice to make the search more efficient, we can
use the numbers in [PATCH <foo>/<bar>] to stop early.
"""
start_id = self.get_number_from_user(user_input)
messages = limit(self.messages_starting_from(start_id), search_limit)
try:
thread = Thread(messages.next())
except StopIteration:
raise FatalError("No message at id '%s' using XOVER")
n_since_last = 0
for message in messages:
if n_since_last > 5:
break
elif thread.should_include(message):
n_since_last = 0
thread.append(message)
else:
n_since_last += 1
else:
raise FatalError('did not find end of series within %s messages', search_limit)
for message in self.xover(start_id - 5, start_id -1):
if thread.should_include(message):
thread.append(message)
return self.mboxify(thread)
def mboxify(self, thread):
"""
Convert a thread into an mbox for application via git-am.
"""
lines = []
for message in thread.in_order():
_, number, msg_id, body = self.conn.body(str(message.number))
# git-am doesn't like empty patches very much, and the 0/X'th patch is
# often not a patch, we skip it here. (TODO, warn the user about this)
if re.search(r" 0+/[0-9]+", message.subject) and not self.is_diff(body):
continue
poster = parseaddr(message.poster)[0]
date = ctime(mktime(parsedate(message.date)))
lines.append("From %s %s" % (poster, date))
lines.append("From: %s" % message.poster)
lines.append("Subject: %s" % message.subject)
lines.append("Date: %s" % message.date)
lines.append("Message-Id: %s" % message.msg_id)
lines.append("Xref: %s %s:%s" % (self.server, self.group, message.number))
lines.append("References: %s" % "\n\t".join(message.references))
lines.append("")
lines += body
lines.append("")
return "\n".join(lines)
def messages_starting_from(self, start_id):
"""
Generate all message headers starting from the given id and working upwards.
"""
while start_id < self.last:
next_id = min(start_id + 20, self.last)
for message in self.xover(start_id, next_id):
yield message
start_id = next_id + 1
def xover(self, begin, end):
"""
Get the headers for the messages with numbers between begin and end.
"""
if begin == end:
return []
_, result = self.conn.xover(str(min(begin, end)), str(max(begin, end)))
result = [Message(int(number), subject, poster, date, msg_id, references) for
(number, subject, poster, date, msg_id, references, size, lines) in result]
return sorted(result, key=lambda x: x.number)
# Open file to write
f = open('samplegroup_fra.xml', 'w')
f.write('<dialog>\n')
f.close()
# Open file for statistics
g = open('stats_message.csv', 'wb')
group_writer = csv.writer(g)
group_writer.writerow(['Group', 'No. of messages', 'Mean authors', 'Max authors'])
g.close()
# Iterate through French groups (only subset at present)
for group in french_groups[]:
# Read number of messages in group
resp, count, first, last, name = s.group(group)
print "Group [" + group + "] has " + count + " articles (" + first + ", " + last + ")"
# Skip empty newsgroups
if count > 0:
# Read items info from group
print "- Reading items"
# DEBUG - ******** THIS NEEDS TO BE REMOVED, IT JUST LOOKS AT LAST 50 MESSAGES TO SAVE TIME FOR NOW *********
#if int(last)-int(first) > 200:
# first = str(int(last)-200)
# print "-- DEBUG: Truncating to (" + first + "," + last + ")"
# DEBUG
resp, items = s.xover(first, last)
class DanskGruppenArchive(object):
"""Class that provides an interface to Dansk-gruppens emails archive on
gmane
"""
def __init__(self, article_cache_size=300, cache_file=None):
"""Initialize local variables"""
# Connect to news.gmane.org
self.nntp = NNTP('news.gmane.org')
# Setting the group returns information, which right now we ignore
self.nntp.group('gmane.comp.internationalization.dansk')
# Keep a local cache in an OrderedDict, transferred across session
# in a pickled version in a file
self.article_cache_size = article_cache_size
self.cache_file = cache_file
if cache_file and path.isfile(cache_file):
with open(cache_file, 'rb') as file_:
self.article_cache = pickle.load(file_)
logging.info('Loaded %i items from file cache',
len(self.article_cache))
else:
self.article_cache = OrderedDict()
def close(self):
"""Quit the NNTP session and save the cache"""
self.nntp.quit()
if self.cache_file:
with open(self.cache_file, 'wb') as file_:
pickle.dump(self.article_cache, file_)
logging.info('Wrote %i items to cache file',
len(self.article_cache))
@property
def last(self):
"""Return the last NNTP ID as an int"""
return self.nntp.group('gmane.comp.internationalization.dansk')[3]
def _get_article(self, message_id):
"""Get an article (cached)
Args:
message_id (int): The NNTP ID of the message
Returns:
list: List of byte strings in the message
"""
# Clear excess cache
if len(self.article_cache) > self.article_cache_size:
self.article_cache.popitem(last=False)
# Check if article is in cache and if not, put it there
if message_id not in self.article_cache:
# nntp.article() returns: response, information
# pylint: disable=unbalanced-tuple-unpacking
_, info = self.nntp.article(message_id)
self.article_cache[message_id] = info
return self.article_cache[message_id]
@staticmethod
def _article_to_email(article):
"""Convert a raw article to an email object
Args:
article (namedtuple): An article named tuple as returned by NNTP
Returns:
email.message: An email message object
"""
# article lines are a list of byte strings
decoded_lines = [line.decode('ascii') for line in article.lines]
article_string = '\n'.join(decoded_lines)
# Make an email object
return email.message_from_string(article_string)
def get_subject(self, message_id):
"""Get the subject of an message
Args:
message_id (int): The NNTP ID of the the message
Returns:
str: The subject of the article
"""
article = self._get_article(message_id)
mail = self._article_to_email(article)
# The subject may be encoded by NNTP, so decode it
return decode_header(mail['Subject'])
def get_body(self, message_id):
"""Get the body of a message
Args:
message_id (int): The NNTP ID of the the message
Returns:
str: The body of the article as a str or None if no body could be
found or succesfully decoded
"""
article = self._get_article(message_id)
mail = self._article_to_email(article)
# Walk parts of the email and look for text/plain content type
for part in mail.walk():
if part.get_content_type() == 'text/plain':
body = part.get_payload(decode=True)
# Find the text encoding from lines like:
# text/plain; charset=UTF-8
# text/plain; charset=utf-8; format=flowed
# Encoding sometimes has "" around it, decode is OK with that
for type_part in part['Content-Type'].split(';'):
if type_part.strip().startswith('charset='):
encoding = type_part.replace('charset=', '')
break
else:
message = 'Looking for the character encoding in the '\
'string "%s" went wrong'
logging.warning(message, part['Content-Type'])
return None
# Decode and return the body
try:
body = body.decode(encoding)
except LookupError:
message = 'Do not know how to handle a body with '\
'charset: %s'
logging.warning(message, encoding)
return None
return body
def get_attachment(self, message_id, filename):
"""Get attachment by filename
Args:
message_id (int): The NNTP ID of the the message
filename (str): The filename for the attachment
Returns:
bytes: The binary content of the attachment
"""
return self.get_attachments(message_id).get(filename)
def get_attachments(self, message_id):
"""Get attachments
Args:
message_id (int): The NNTP ID of the the message
Returns:
dict: Dict with attachments where keys are filenames and values are
their binary content
"""
article = self._get_article(message_id)
mail = self._article_to_email(article)
attachments = {}
# Walk parts of the email and look for application/octet-stream
# content type
for part in mail.walk():
content_disp = part['Content-Disposition']
if not (content_disp and content_disp.startswith('attachment')):
continue
# Get the filename from a line like: Content-Disposition:
# attachment; filename="hitori.master.da.podiff"
filename = None
for disp_part in content_disp.split(';'):
if disp_part.strip().startswith('filename='):
filename = disp_part.strip().replace('filename=', '')
# Strip " from filename
filename = filename.strip('"')
if filename is None:
message = 'Unable to extract filename from '\
'Content-Disposition: %s'
logging.warning(message, part['Content-Disposition'])
raise Exception('Unable to extract filename')
attachments[filename] = part.get_payload(decode=True)
return attachments
import nntplib
from nntplib import NNTP
s = NNTP('news.gmane.org')
resp, count, first, last, name = s.group('gmane.comp.python.committers')
print ('Group', name, 'has', count, 'articles, range', first, 'to', last)
resp, subs = s.xhdr('subject',first)
for id, sub in subs[-10:]: print (id, sub)
try :
for id, sub in subs[-10:]: print (id, sub)
except TypeError as e:
print(e)
print(s.quit())
# NNTP.
import textwrap
import re
# print strftime('%y%m%d')
# start = localtime(time()-60*60*24*365)
# print strftime('%y%m%d',start)
class SimpleWebSource:
def __init__(self, url, titlePattern, bodyPattern):
self.url = url
self.titlePattern = re.compile(titlePattern)
self.bodyPattern = re.compile(bodyPattern)
def getItems(self):
text = urlopen(self.url).read()
titles = self.titlePattern.findall(text)
bodies = self.bodyPattern.findall(text)
for title in titles:
print title[1].decode('utf-8')
print bodies
bbc_url = 'http://news.sina.com.cn'
bbc_title = r'<a target="_blank" href="(.*?)">(.*?)</a>'
bbc_body = r'(?s)</a>\s*<br />\s*(.*?)\s*<'
# bbc = SimpleWebSource(bbc_url,bbc_title,bbc_body)
# bbc.getItems()
server = NNTP('129.69.1.59')
server.group('camp.lang.python')[0]
class DownloadSpots(object):
def __init__(self):
try:
self.news = NNTP(NEWS_SERVER, NNTP_PORT, NNTP_USERNAME,
NNTP_PASSWORD)
except NNTPTemporaryError as e:
raise SpotError('NNTP', e)
except socket.error as e:
raise SpotError('Connection', e)
self.conn = sqlite3.connect(NEWS_SERVER + '.db')
self.conn.row_factory = sqlite3.Row
self.cur = self.conn.cursor()
self.cur.executescript('''\
PRAGMA synchronous = OFF;
PRAGMA journal_mode = MEMORY;
PRAGMA temp_store = MEMORY;
PRAGMA count_changes = OFF;
''')
def __del__(self):
print 'quit!'
if hasattr(self, 'news'):
self.news.quit()
if hasattr(self, 'cur'):
self.cur.close()
if hasattr(self, 'conn'):
self.conn.close()
def make_table(self):
sql = '''\
CREATE TABLE IF NOT EXISTS spots (
id int PRIMARY KEY,
full_id str,
cat int,
title str,
poster str,
date int,
size int,
erotiek int,
subcats str,
modulus int,
keyid int,
c_count int
);
CREATE TABLE IF NOT EXISTS comments (
id int PRIMARY KEY,
full_id str,
spot_full_id str
);
CREATE INDEX IF NOT EXISTS spots_full_id_index on spots(full_id);
'''
self.cur.executescript(sql)
self.conn.commit()
def download_detail(self, id):
'''Get information about a spot.
Args:
id (int/string): the nntp id of the spot. Can be:
- a short id: 123456
- a long id: '*****@*****.**'
Returns:
a dict with the following keys:
- nzb: a list of nntp id's of the nzb file
- image: the location of the image file: url or nntp id
- website: url of the website
- desc: a description of the spot
- title2: the title of the spot
(is called title2 to avoid a conflict)
'''
id = str(id)
self.news.group('free.pt')
print id
head = self.news.head(id)[-1]
xmltext = ''.join(item[7:] for item in head if
item.startswith('X-XML:'))
xml = etree.XML(xmltext)
xmldict = defaultdict(list)
xmldict['nzb'] = [i.text for i in xml.find('.//NZB').iter('Segment')]
imgfind = xml.find('.//Image')
if imgfind is not None:
if imgfind.find('.//Segment') is not None:
xmldict['image'] = imgfind.find('.//Segment').text
else:
xmldict['image'] = imgfind.text
else:
xmldict['image'] = None
webfind = xml.find('.//Website')
if webfind is not None:
xmldict['website'] = webfind.text
else:
xmldict['website'] = None
xmldict['desc'] = bb2html('\n'.join(self.news.body(id)[-1]))
xmldict['title2'] = xml.find('.//Title').text
print xmldict
return xmldict
def download_image(self, article, imgnr):
'''Download and save an image file.
Args:
article: Location of the file, can be:
- None
- an url: 'http://url.to.image.ext'
- a nntp id: '*****@*****.**'
imgnr: filename of the saved image
'''
print imgnr
file = 'temp/%s.picture' % imgnr
if article is None:
shutil.copy('none.png', file)
elif article.startswith('http'):
if 'imgchili.com' in article:
article = re.sub(r't([0-9]\.imgchili)', r'i\1', article)
try:
print urllib.urlretrieve(article, file)
except:
print 'Image download error'
shutil.copy('none.png', file)
elif '@' in article:
article = '<%s>' % article
print article
try:
self.news.group('alt.binaries.ftd')
data = ''.join(self.news.body(article)[-1])
except NNTPTemporaryError as e:
shutil.copy('none.png', file)
raise SpotError('NNTP', e)
else:
data = data.replace('=A', '\x00').replace('=B', '\r') \
.replace('=C', '\n').replace('=D', '=')
with open(file, 'wb') as f:
f.write(data)
else:
shutil.copy('none.png', file)
def download_nzb(self, articles, title):
'''Download and save a nzb file.
Args:
articles: a list of nntp id's: ['*****@*****.**']
title (string): the filename of the saved nzb file
(must be already escaped)
'''
print 'download_nzb'
file = 'temp/%s.nzb' % title
self.news.group('alt.binaries.ftd')
print articles
data = ''.join(''.join(self.news.body('<%s>' % article)[-1]) for
article in articles)
data = data.replace('=A', '\x00').replace('=B', '\r') \
.replace('=C', '\n').replace('=D', '=')
data = zlib.decompress(data, -zlib.MAX_WBITS)
with open(file, 'wb') as f:
f.write(data)
def update_spots(self):
'''Download new spots and save them to the database.
Yields:
total: total amount of spots to be downloaded
subtotal: amount of spots already done
'''
print 'Opening database...',
self.make_table()
self.cur.execute('SELECT count(id) FROM spots')
oude_lengte = self.cur.fetchone()[0]
print 'Done'
print oude_lengte, 'spots in database'
print 'Connecting...',
last = int(self.news.group('free.pt')[3])
print 'Done'
if oude_lengte:
self.cur.execute('SELECT max(id) FROM spots')
current = self.cur.fetchone()[0]
else:
current = last - (NR_OF_SPOTS * 100)
delta = 2000
total = last - current
print 'Current:', current
print 'Last:', last
print total, 'spots op te halen'
yield total
if current >= last:
print 'Geen nieuwe spots!'
yield 'end'
return
print 'Getting new spots...',
self.cur.execute('DROP INDEX IF EXISTS spots_full_id_index')
self.conn.commit()
for i in xrange(0, total, delta):
for (id, subject), (id, fromline), (id, full_id) in \
zip(*(self.news.xhdr(header, '%s-%s'
% (current + 1 + i, current + delta + i))[-1] for
header in ('subject', 'from', 'Message-ID'))):
subject = re.sub('^[\t]', '', subject).split('\t')[0]
if subject[:7] == 'DISPOSE':
continue
subject = subject.decode('latin-1')
fromline = fromline.decode('latin-1')
subject = re.sub('\s\|\s\w+$', '', subject)
poster = re.sub('(\t[^\t]+\t)?([^\s]+)\s<.+$', r'\2', fromline)
try:
info = re.findall(r'<[^>]+>', fromline)[0].split('@')
modulus = info[0]
info = info[1].split('.')[:4]
date = info[3]
except IndexError:
continue
if info[0][1] == '7':
modulus = modulus.split('.')[0][1:]
else:
modulus = 0
subcats = '|'.join(info[0][2:][x:x + 3] for x in
xrange(0, len(info[0][2:]), 3))
erotiek = 0
if info[0][0] == '1':
for genre in re.findall('d[0-9]{2}', subcats):
if ((23 <= int(genre[1:]) <= 26) or
(72 <= int(genre[1:]) <= 89)):
erotiek = 1
break
sql = ('INSERT INTO spots (id, full_id, cat, title, poster, date, '
'size, erotiek, subcats, modulus, keyid, c_count) '
'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)')
self.cur.execute(sql, (id, full_id, info[0][0], subject, poster,
date, info[1], erotiek, subcats, modulus,
info[0][1]))
self.conn.commit()
yield int(id) - current # subtotal
self.cur.execute('CREATE INDEX spots_full_id_index on spots(full_id)')
self.conn.commit()
print 'Spots klaar!'
self.cur.execute('SELECT count(id) FROM spots')
lengte = self.cur.fetchone()[0]
print lengte - oude_lengte, 'nieuwe spots!'
self.news.quit()
yield 'end'
def update_comments(self):
'''Download new comments and save them to the database.
Yields:
total: total amount of comments to be downloaded
subtotal: amount of comments already done
'''
print 'comments'
print 'Opening database...',
self.make_table()
self.cur.execute('SELECT count(id) FROM comments')
oude_lengte = self.cur.fetchone()[0]
print 'Done'
print oude_lengte, 'comments in database'
print 'Connecting...',
last = int(self.news.group('free.usenet')[3])
print 'Done'
if oude_lengte:
self.cur.execute('SELECT max(id) FROM comments')
current = self.cur.fetchone()[0]
print current
else:
current = last - (NR_OF_SPOTS * 500)
delta = 2000
total = last - current
print 'Current:', current
print 'Last:', last
print total, 'comments op te halen'
yield total
if current >= last:
print 'Geen nieuwe comments!'
yield 'end'
return
print 'begin reacties', time.time()
for i in xrange(0, total, delta):
for (id, ref), (id, msgid) in \
zip(*(self.news.xhdr(header, '%s-%s'
% (current + 1 + i, current + delta + i))[-1] for
header in ('References', 'Message-ID'))):
self.cur.execute('INSERT INTO comments VALUES (?, ?, ?)',
(id, msgid, ref))
self.cur.execute('UPDATE spots SET c_count = c_count + 1 WHERE full_id = ?',
(ref,))
yield int(id) - current # subtotal
self.conn.commit()
print 'Reacties klaar'
self.cur.execute('SELECT count(id) FROM comments')
lengte = self.cur.fetchone()[0]
print lengte - oude_lengte, 'nieuwe comments!'
self.news.quit()
yield 'end'
def show(self, search_col, op, search, search_rest='', limit=NR_OF_SPOTS,
columns='*'):
'''Perform a sql query.
Args:
search_col: the column to use in the where clause
op: the operator to use in the where clause
search: the search value to use in the where clause
search_rest: additional conditions for the where clause
limit: the number of rows to select
columns: the columns to select
Returns:
a list of tuples with the results of the sql query
'''
self.make_table()
sql = '''\
SELECT %s FROM spots
WHERE %s %s :search %s
ORDER BY id DESC
LIMIT %s
''' % (columns, search_col, op, search_rest, limit)
results = map(Spot, self.cur.execute(sql, (search,)))
print len(results)
return results
__author__ = 'sks'
from nntplib import NNTP
from time import strftime, time, localtime
day = 24 * 60 * 60
yesterday = localtime(time() - day)
date = strftime('%y%m%d', yesterday)
hour = strftime('%H%M%S', yesterday)
servername = 'data.eastmoney.com'
group = 'comp.lang.python.announce'
server = NNTP(servername)
print(server.group(group)[0])
# to be continued
#ids = server.newnews()
#!/usr/local/bin/python3
from nntplib import NNTP
server = NNTP('news.aioe.org')
resp, count, first, last, name = server.group('sci.logic')
print('Group', name, 'has', count, 'articles, range', first, 'to', last)
resp, subs = server.xhdr('subject', str(first) + '-' + str(last))
for id, text in subs[-10:]:
print(id, text)
server.quit()
import sqlite3
from nntplib import NNTP
group = 'alt.binaries.movies.x264'
chunk_size = 100000
sql = sqlite3.connect('usenet.db')
print "About to start dumping articles from " + group
serv = NNTP('news.astraweb.com', 119, 'arealmuto', 'stock1114')
resp = serv.group(group)
count = int(resp[1])
first = int(resp[2])
last = int(resp[3])
print "There are " + str(count) + " articles to get"
print "First: " + str(first)
print "Last: " + str(last)
print "Using chunks size of: " + str(chunk_size)
print "It should take " + str(count/chunk_size) + " requests to finish"
id = int(first)
i = 0
master_list = []
while id < last:
print str(i) +": Getting id's " + str(id) + " - " + str(id + chunk_size)
resp, list = serv.xover(str(id), str(id+ chunk_size))
print "Done fetching"
print "Adding to master list"
for line in list:
#/usr/bin/python
# *-*coding:utf-8 *-*
#python NNTP
# group组返回响应信息
from nntplib import NNTP
with NNTP('news.gmane.org') as n:
print(n.group('gmane.comp.python.committers'))
server = NNTP('web.aioe.org')
print(server.group('comp.lang.python.announce')[0])
#('211 4419 1 4419 gmane.comp.python.committers', 4419, 1, 4419, 'gmane.comp.python.committers')
#211 100 3923 4023 comp.lang.python.announce
'''
('211 4371 1 4371 gmane.comp.python.committers', 4371, 1, 4371, 'gmane.comp.python.committers')
411 开头意味着服务器没有这个组
nntplib.NNTPTemporaryError: 411 No such group comp.lang.python.announce
211开头 基本上意味着服务器拥有你所请求的组
('211 98 3911 4008 comp.lang.python.announce', 98, 3911, 4008, 'comp.lang.python.announce')
还有可能得到一个带有类似错误信息的的异常 如果引发了异常 可能是服务器的名字写错了
另外一种可能是在创建服务器对象和调用group方法之间 '超时了'， 服务器允许用户保持连接的时间很短(比如10秒)
如果输入速度不够快 把代码放到脚本里， 或着将服务器对象的创建和方法的调用放在同一行内 (以分号隔开)
from nntplib import NNTP
servername = 'news.foo.bar'
group = 'comp.lang.python.announce'
server = NNTP(servername)
howmany = 10
resp, count, first, last, name = server.group(group)
start = last - howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
subject = over['subject']
resp, info = server.body(id)
print(subject)
print('-' * len(subject))
for line in info.lines:
print(line.decode('latin1'))
print()
server.quit()
listonly = 0
showhdrs = ['From', 'Subject', 'Date', 'Newsgroups', 'Lines']
try:
import sys
servername, groupname, showcount = sys.argv[1:]
showcount = int(showcount)
except:
servername = 'news.gmane.org'
groupname = 'gmane.comp.python.general' # cmd line args or defaults
showcount = 10 # show last showcount posts
# connect to nntp server
print 'Connecting to', servername, 'for', groupname
from nntplib import NNTP
connection = NNTP(servername)
(reply, count, first, last, name) = connection.group(groupname)
print '%s has %s articles: %s-%s' % (name, count, first, last)
# get request headers only
fetchfrom = str(int(last) - (showcount-1))
(reply, subjects) = connection.xhdr('subject', (fetchfrom + '-' + last))
# show headers, get message hdr+body
for (id, subj) in subjects: # [-showcount:] if fetch all hdrs
print 'Article %s [%s]' % (id, subj)
if not listonly and raw_input('=> Display?') in ['y', 'Y']:
reply, num, tid, list = connection.head(id)
for line in list:
for prefix in showhdrs:
if line[:len(prefix)] == prefix:
print line[:80]; break
class DanskGruppenArchive(object):
"""Class that provides an interface to Dansk-gruppens emails archive on
gmane
"""
def __init__(self, article_cache_size=300, cache_file=None):
"""Initialize local variables"""
# Connect to news.gmane.org
self.nntp = NNTP('news.gmane.org')
# Setting the group returns information, which right now we ignore
self.nntp.group('gmane.comp.internationalization.dansk')
# Keep a local cache in an OrderedDict, transferred across session
# in a pickled version in a file
self.article_cache_size = article_cache_size
self.cache_file = cache_file
if cache_file and path.isfile(cache_file):
with open(cache_file, 'rb') as file_:
self.article_cache = pickle.load(file_)
logging.info('Loaded %i items from file cache',
len(self.article_cache))
else:
self.article_cache = OrderedDict()
def close(self):
"""Quit the NNTP session and save the cache"""
self.nntp.quit()
if self.cache_file:
with open(self.cache_file, 'wb') as file_:
pickle.dump(self.article_cache, file_)
logging.info('Wrote %i items to cache file',
len(self.article_cache))
@property
def last(self):
"""Return the last NNTP ID as an int"""
return self.nntp.group('gmane.comp.internationalization.dansk')[3]
def _get_article(self, message_id):
"""Get an article (cached)
Args:
message_id (int): The NNTP ID of the message
Returns:
list: List of byte strings in the message
"""
# Clear excess cache
if len(self.article_cache) > self.article_cache_size:
self.article_cache.popitem(last=False)
# Check if article is in cache and if not, put it there
if message_id not in self.article_cache:
# nntp.article() returns: response, information
# pylint: disable=unbalanced-tuple-unpacking
_, info = self.nntp.article(message_id)
self.article_cache[message_id] = info
return self.article_cache[message_id]
@staticmethod
def _article_to_email(article):
"""Convert a raw article to an email object
Args:
article (namedtuple): An article named tuple as returned by NNTP
Returns:
email.message: An email message object
"""
# article lines are a list of byte strings
decoded_lines = [line.decode('ascii') for line in article.lines]
article_string = '\n'.join(decoded_lines)
# Make an email object
return email.message_from_string(article_string)
def get_subject(self, message_id):
"""Get the subject of an message
Args:
message_id (int): The NNTP ID of the the message
Returns:
str: The subject of the article
"""
article = self._get_article(message_id)
mail = self._article_to_email(article)
# The subject may be encoded by NNTP, so decode it
return decode_header(mail['Subject'])
def get_body(self, message_id):
"""Get the body of a message
Args:
message_id (int): The NNTP ID of the the message
Returns:
str: The body of the article as a str or None if no body could be
found or succesfully decoded
"""
article = self._get_article(message_id)
mail = self._article_to_email(article)
# Walk parts of the email and look for text/plain content type
for part in mail.walk():
if part.get_content_type() == 'text/plain':
body = part.get_payload(decode=True)
# Find the text encoding from lines like:
# text/plain; charset=UTF-8
# text/plain; charset=utf-8; format=flowed
# Encoding sometimes has "" around it, decode is OK with that
for type_part in part['Content-Type'].split(';'):
if type_part.strip().startswith('charset='):
encoding = type_part.replace('charset=', '')
break
else:
message = 'Looking for the character encoding in the '\
'string "%s" went wrong'
logging.warning(message, part['Content-Type'])
return None
# Decode and return the body
try:
body = body.decode(encoding)
except LookupError:
message = 'Do not know how to handle a body with '\
'charset: %s'
logging.warning(message, encoding)
return None
return body
def get_attachment(self, message_id, filename):
"""Get attachment by filename
Args:
message_id (int): The NNTP ID of the the message
filename (str): The filename for the attachment
Returns:
bytes: The binary content of the attachment
"""
return self.get_attachments(message_id).get(filename)
def get_attachments(self, message_id):
"""Get attachments
Args:
message_id (int): The NNTP ID of the the message
Returns:
dict: Dict with attachments where keys are filenames and values are
their binary content
"""
article = self._get_article(message_id)
mail = self._article_to_email(article)
attachments = {}
# Walk parts of the email and look for application/octet-stream
# content type
for part in mail.walk():
content_disp = part['Content-Disposition']
if not (content_disp and content_disp.startswith('attachment')):
continue
# Get the filename from a line like: Content-Disposition:
# attachment; filename="hitori.master.da.podiff"
filename = None
for disp_part in content_disp.split(';'):
if disp_part.strip().startswith('filename='):
filename = disp_part.strip().replace('filename=', '')
# Strip " from filename
filename = filename.strip('"')
if filename is None:
message = 'Unable to extract filename from '\
'Content-Disposition: %s'
logging.warning(message, part['Content-Disposition'])
raise Exception('Unable to extract filename')
attachments[filename] = part.get_payload(decode=True)
return attachments
from nntplib import NNTP
#server = NNTP('news.easynews.com')
#server = NNTP('news.giganews.com')
#server = NNTP('news.mozilla.org')
server = NNTP('news.kornet.net')
print server.group('comp.lang.python.announce')[0]
"""
| group(self, name)
| Process a GROUP command. Argument:
| - group: the group name
| Returns:
| - resp: server response if successful
| - count: number of articles (string)
| - first: first article number (string)
| - last: last article number (string)
| - name: the group name
"""
group = server.group('han.test')
print repr(group)
first = group[2]
last = group[3]
print "first ", first
print "last ", last
i = 0
for id in range(int(first), int(last)):
i += 1
print(server.article(str(id)))
class Client(Context):
def __init__(self,
log,
hostname,
port=None,
username=None,
password=None,
*,
debuglevel=None,
**timeout):
self.log = log
self.hostname = hostname
self.port = port
self.username = username
self.password = password
self.debuglevel = debuglevel
self.timeout = timeout
Context.__init__(self)
self.connect()
def connect(self):
address = net.format_addr((self.hostname, self.port))
self.log.write("Connecting to {}\n".format(address))
if self.port is None:
port = ()
else:
port = (self.port, )
self.connect_time = time.monotonic()
self.nntp = NNTP(self.hostname, *port, **self.timeout)
with ExitStack() as cleanup:
cleanup.push(self)
if self.debuglevel is not None:
self.nntp.set_debuglevel(self.debuglevel)
self.log.write("{}\n".format(self.nntp.getwelcome()))
if self.username is not None:
self.log.write("Logging in as {}\n".format(self.username))
with self.handle_abort():
self.nntp.login(self.username, self.password)
self.log.write("Logged in\n")
cleanup.pop_all()
def body(self, id, *pos, **kw):
id = "<{}>".format(id)
retry = 0
while True:
try:
with self.handle_abort():
self.nntp.body(id, *pos, **kw)
break
except failure_responses as err:
[code, *msg] = err.response.split(maxsplit=1)
if code == "400":
[msg] = msg or (None, )
if not msg:
msg = "Server shut down connection"
elif code[1] == "0" and not retry:
msg = err.response
else:
raise
self.log.write(msg + "\n")
self.log_time()
if retry >= 60:
raise TimeoutError()
self.close()
time.sleep(retry)
if not retry:
start = time.monotonic()
self.connect()
if retry:
retry *= 2
else:
retry = time.monotonic() - start
if retry <= 0:
retry = 0.5
def group(self, *pos, **kw):
with self.handle_abort():
return self.nntp.group(*pos, **kw)
def over(self, *pos, **kw):
with self.handle_abort():
return self.nntp.over(*pos, **kw)
def hdr(self, *pos, **kw):
with self.handle_abort():
return self.nntp.xhdr(*pos, **kw)
@contextmanager
def handle_abort(self):
try:
yield
except failure_responses:
raise # NNTP connection still intact
except:
# Protocol is disrupted so abort the connection straight away
self.close()
raise
def close(self):
if not self.nntp:
return
# Ignore failure of inappropriate QUIT command
with suppress(NNTPError), self.nntp:
pass
self.nntp = None
def log_time(self):
# TODO: time duration formatter
self.log.write("Connection lasted {:.0f}m\n".format(
(time.monotonic() - self.connect_time) / 60))
class NewsWatcher(MessageWatcher):
def __init__(self, server, groups,
user=None, pw=None, port=None, tag=None):
MessageWatcher.__init__(self)
self.server = server
self.groups = groups
self.nntp = None # the NNTP connection object
self.user = user
self.pw = pw
self.port = port
self.tag = tag
self.last = {}
self.timeout = None
self.pollInterval = 60
self.debug = 0
def __repr__(self):
return "<NewsWatcher %s:%s (%s)>" % (self.server, self.port,
",".join(self.groups))
def __getstate__(self):
d = MessageWatcher.__getstate__(self)
d['nntp'] = None # just in case
return d
def start(self):
port = self.port
if not port:
port = NNTP_PORT
self.nntp = NNTP(self.server, port, self.user, self.pw,
readermode=1)
# only look for messages that appear after we start. Usenet is big.
if not self.last: # only do this the first time
for g in self.groups:
resp, count, first, last, name = self.nntp.group(g)
self.last[g] = int(last)
if self.debug: print "last[%s]: %d" % (g, self.last[g])
self.timeout = gtk.timeout_add(self.pollInterval*1000,
self.doTimeout)
def stop(self):
self.nntp.quit()
self.nntp = None
if self.timeout:
gtk.timeout_remove(self.timeout)
self.timeout = None
def doTimeout(self):
self.poll()
return gtk.TRUE # keep going
def poll(self):
#print "polling", self
for g in self.groups:
resp, count, first, last, name = self.nntp.group(g)
for num in range(self.last[g]+1, int(last)+1):
try:
resp, num, id, lines = self.nntp.article("%d" % num)
except NNTPError:
continue
name = "%s:%d" % (g, int(num))
if self.debug: print "got", name
if self.tag:
if not filter(lambda line, tag=tag: line.find(tag) != -1,
lines):
continue
self.parseMessage(name, name, time.time(), lines)
self.last[g] = int(last)
from nntplib import NNTP
server = NNTP('news2.neva.ru')
print server.group('alt.sex.telephone')
