def gmane(self): try: gmane = NNTP("news.gmane.org") except: return self.gmane gmane.group(self.group) return gmane
def getItems(self): server = NNTP(self.servername) group = server.group(self.group) first = group[2] last = str(int(group[2]) + 10) #last = str(int(group[2]) + 2) for id in range(int(first), int(last)): lines = server.article(str(id))[3] message = message_from_string('\n'.join(lines)) title = message['subject'] #print(repr(dir(message))) print(message.get_content_type()) if 'text' in message.get_content_type(): body = message.get_payload() else: body = 'Body is not text' if 'ybegin' in message.get_payload(): body = 'hmm' if message.is_multipart(): #body = body[0] body = 'Multi part is not implemented yet' yield NewsItem(title, body) server.quit()
class NNTPSource(Source): PREFIX = 'dispose' def __init__(self, serverName, group, howmany): super().__init__() self.serverName = serverName self.type = Source.NNTP self.group = group self.howmany = howmany self.server = NNTP(self.serverName) def getNewsItem(self): resp, count, first, last, name = self.server.group(self.group) resp = resp.split(' ')[0] if resp == '211': # 正常响应 start = last - self.howmany + 1 resp, overviews = self.server.over((start, last)) for id, over in overviews: title = decode_header(over['subject']) resp, info = self.server.body(id) body = '\n'.join( line.decode('latin') for line in info.lines) + '\n\n' # 使用生成器推导,转字符串 yield NewsItem(title, body, self.NNTP) else: yield None self.server.quit() def getDisposeName(self): return self.PREFIX + self.type pass def disposeNNTP(self, news): print('NNTP新闻生产完毕') pass
def main(): s = NNTP(settings.nntp_hostname, settings.nntp_port) resp, groups = s.list() # check all of the groups, just in case for group_name, last, first, flag in groups: resp, count, first, last, name = s.group(group_name) print "\nGroup", group_name, 'has', count, 'articles, range', first, 'to', last resp, subs = s.xhdr('subject', first + '-' + last) for id, sub in subs[-10:]: print id, sub s.quit()
def get_items(self): #新闻生成器 server = NNTP(self.servername) resp, count, first, last, name = server.group(self.group) #新闻组信息列表 start = last - self.howmany + 1 resp, overviews = server.over((start, last)) for id, over in overviews: title = decode_header(over['subject']) resp, info = server.body(id) body = '\n'.join(line.decode() for line in info.lines) + '\n\n' yield NewsItem(title, body) server.quit()
def use_NNTP(): for server in NNTP_server_list: print server server = raw_input('Which NNTP_server do you want to go?\n') while True: if server not in NNTP_server_list: print 'invalid srever name,try again!' server = raw_input() else: break s = NNTP(server) print "s = NNTP('news.newsfan.net') is OK!" (resp, lst) = s.list() while True: for i, elem in enumerate(lst): print i, elem[0].decode('gbk') num = raw_input('Which group do you want to go?\n') try: rsp, ct, first, last, grp = s.group(lst[int(num)][0]) print "Article's range is:%s to %s." % (first, last) (resp, subs) = s.xhdr('subject', (str(first) + '-' + str(last))) except: print format_exc() print 'invalid input!try again!!!' sleep(3) continue for subject in subs: try: print subject[0], subject[1].decode('gbk') except: print subject[0], subject[1] while True: try: number = raw_input('Which article do you want to read?\n') if number == 'q': break f = open("NNTPfile", 'w') (reply, num, id, list) = s.body(str(number), f) f = open("NNTPfile", 'r') for eachLine in f: try: print eachLine.decode('gbk'), except: print eachLine f.close() print 'Press any to continue...(Press q to return...)' if raw_input() == 'q': break except: print format_exc() print 'invalid input!try again!!!' sleep(3) s.quit() return
def get_items(self): server = NNTP(self.servername) resp, count, first, last, name = server.group(self.group) start = last - self.howmany + 1 resp, overviews = server.over((start, last)) for id, over in overviews: title = decode_header(over["subject"]) resp, info = server.body(id) body = "\n".join(line.decode("latin") for line in info.lines) + "\n\n" yield NewsItem(title, body) server.quit()
def get_items(self): server = NNTP(self.servername) resp, count, first, last, name = server.group(self.group) start = last - self.howmany + 1 resp, overviews = server.over((start, last)) for id, over in overviews: title = decode_header(over['subject']) resp, info = server.body(id) body = '\n'.join(line.decode('latin') for line in info.lines) + '\n\n' yield NewsItem(title, body) server.quit()
def get_items(self): server = NNTP(self.servername) _, count, first, last, name = server.group(self.group) start = last - self.how_many + 1 _, overviews = server.over((start, last)) for ID, over in overviews: title = decode_header(over['subject']) _, info = server.body(ID) body = '\n'.join(line.decode('latin') for line in info.lines) yield NewsItem(title, body) server.quit()
def download_group(name): if not os.path.exists(name): os.mkdir(name) s = NNTP('news.gmane.org') resp, count, first, last, name = s.group(name) print 'Group', name, 'has', count, 'articles, range', first, 'to', last resp, subs = s.xhdr('subject', first + '-' + last) for id, sub in subs: print id with open(os.path.join(name, str(id)), 'wb') as fp: pprint.pprint(s.article(id), stream=fp)
def get_items(self): server = NNTP(self.servername) #服务器响应、新闻组包含的消息数、第一条和最后一条消息编号、新闻组名称 resp, count, first, last, name = server.group(self.group) #确定要获取的文章编号区间的起始位置 start = last - self.howmany + 1 resp, overviews = server.over((start, last)) for id, over in overviews: title = decode_header(over['subject']) resp, info = server.body(id) body = '\n'.join(line.decode('latin') for line in info.lines) + '\n\n' yield NewsItem(title, body) server.quit()
def getItems(self): # yesterday = localtime(time()-self.window*day) # date = strftime('%y%m%d',yesterday) # time = strftime('%H%M%S',yesterday) # create a nntp server s = NNTP(self.servername) resp,count,first,last,name = s.group(self.groupname) resp,overviews = s.over((last-1,last)) for num,over in overviews: title = over.get('subject') resp,body = s.body(num) # create a generator to iterate news if title and body: yield NewsItem(title,body) s.quit()
def getItems(self): server = NNTP(self.servername) (resp, count, first, last, name) = server.group(self.group) (resp, subs) = server.xhdr('subject', (str(first) + '-' + str(last))) for subject in subs[-10:]: title = subject[1] (reply, (num, id, list)) = server.body(subject[0]) # list是一个列表,但是是bytes编码的,需要把每一个元素都解码成string body = [] #print(list) for l in list: body.append(l.decode('gbk')) # 注意,这里用utf-8解码会出现解码错误 #print(body) body = ''.join(body) yield NewsItem(title, body) server.quit()
def getItems(self): server = NNTP(self.servername) (resp, count, frist, last, name) = server.group(self.group) (resp, subs) = server.xhdr('subject', (str(frist) + '-' + (last))) for subject in subs[-10:]: title = subject[1] (reply, num, id, list) = server.body(subject[0]) body = ''.join(list) #print(num) #186919 #print(title) #Re: Find out which module a class came from #print(''.join(list))#prano wrote:> But for merely ordinary obfuscation caused by poor... yield NewsItem(title, body) server.quit()
def get_items(self): for servername in KNOWN_NNTP_SERVERS: try: server = NNTP(servername) resp, count, first, last, name = server.group(self.group) start = last - self.howmany + 1 resp, overviews = server.over((start, last)) for id, over in overviews: title = decode_header(over['subject']) resp, info = server.body(id) body = '\n'.join( line.decode('latin1') for line in info.lines) + '\n\n' yield NewsItem(title, body, "NNTP NewsGroup " + self.group) server.quit() break except: continue return []
def getItems(self): server = NNTP(self.servername) _, count, first, last, name = server.group(self.group) _, subs = server.xhdr( 'subject', (str(first) + '-' + str(last))) for sub in subs[:10]: # default last ten id = sub[0] lines = server.article(id)[3] message = message_from_string('\n'.join(lines)) title = message['subject'] body = message.get_payload() if message.is_multipart(): body = body[0] yield NewsItem(title, body) server.quit()
class NNTPConnector(BaseConnector): @logit(log,'fetch') def fetch(self): """ Fetches all the messages for a given news group uri and return Fetched staus depending on the success and faliure of the task """ try: #eg. self.currenturi = nntp://msnews.microsoft.com/microsoft.public.exchange.setup #nntp_server = 'msnews.microsoft.com' #nntp_group = 'microsoft.public.exchange.setup' self.genre = 'review' try: nntp_server = urlparse(self.currenturi)[1] except: log.exception(self.log_msg("Exception occured while connecting to NNTP server %s"%self.currenturi)) return False nntp_group = urlparse(self.currenturi)[2][1:] self.server = NNTP(nntp_server) try: self.__updateParentSessionInfo() resp, count, first, last, name = self.server.group(nntp_group) last_id = int(last) first_id = self.__getMaxCrawledId(last_id)+1 log.debug("first_id is %d:"%first_id) log.debug("last_id is %d:"%last_id) if last_id >= first_id: resp, items = self.server.xover(str(first_id), str(last_id)) log.debug(self.log_msg("length of items:%s"%str(len(items)))) for self.id, self.subject, self.author, self.date, self.message_id,\ self.references, size, lines in items: self.__getMessages(self.task.instance_data['uri']) self.server.quit() return True except: log.exception(self.log_msg("Exception occured in fetch()")) self.server.quit() return False except Exception,e: log.exception(self.log_msg("Exception occured in fetch()")) return False
def getItems(self): """ 书中原例getItems()方法 返回 nntplib.NNTPTemporaryError: 480 NEWNEWS command disabled by administrator #480管理员禁用NEWNEWS命令 def getItems(self): start = localtime(time() - self.window*day) date = strftime('%y%m%d', start) hour = strftime('%H%M%S', start) server = NNTP(self.servername) ids = server.newnews(self.group, date, hour)[1] for id in ids: lines = serverarticle(id)[3] message = message_from_string('\n'.join(lines)) title = message['subject'] body = message.get_payload() if message.is_multipat(): body = body[0] yield NewsItem(title, body) server.quit() """ server = NNTP(self.servername) (resp, count, frist, last, name) = server.group(self.group) (resp, subs) = server.xhdr('subject', (str(frist) + '-' + (last))) for subject in subs[-10:]: title = subject[1] (reply, num, id, list) = server.body(subject[0]) body = ''.join(list) #print(num) #186919 #print(title) #Re: Find out which module a class came from #print(''.join(list))#prano wrote:> But for merely ordinary obfuscation caused by poor... yield NewsItem(title, body) server.quit()
class NewsGrep: def __init__(self,server,username,password): self.server = NNTP(server, 119,username,password) def __del__(self): pass def __str__(self): pass def list(self): resp, groups = self.server.list() for group, last, first, flag in groups: print group def ls(self,group_name): resp, count, first, last, name = self.server.group(group_name) print 'Group', name, 'has', count, 'articles, range', first, 'to', last resp, subs = self.server.xhdr('subject', first + '-' + last) for id, sub in subs[-10:]: print id, sub
class NewsGrep: def __init__(self, server, username, password): self.server = NNTP(server, 119, username, password) def __del__(self): pass def __str__(self): pass def list(self): resp, groups = self.server.list() for group, last, first, flag in groups: print group def ls(self, group_name): resp, count, first, last, name = self.server.group(group_name) print 'Group', name, 'has', count, 'articles, range', first, 'to', last resp, subs = self.server.xhdr('subject', first + '-' + last) for id, sub in subs[-10:]: print id, sub
class Client(Context): def __init__(self, log, hostname, port=None, username=None, password=None, *, debuglevel=None, **timeout): self.log = log self.hostname = hostname self.port = port self.username = username self.password = password self.debuglevel = debuglevel self.timeout = timeout Context.__init__(self) self.connect() def connect(self): address = net.format_addr((self.hostname, self.port)) self.log.write("Connecting to {}\n".format(address)) if self.port is None: port = () else: port = (self.port,) self.connect_time = time.monotonic() self.nntp = NNTP(self.hostname, *port, **self.timeout) with ExitStack() as cleanup: cleanup.push(self) if self.debuglevel is not None: self.nntp.set_debuglevel(self.debuglevel) self.log.write("{}\n".format(self.nntp.getwelcome())) if self.username is not None: self.log.write("Logging in as {}\n".format(self.username)) with self.handle_abort(): self.nntp.login(self.username, self.password) self.log.write("Logged in\n") cleanup.pop_all() def body(self, id, *pos, **kw): id = "<{}>".format(id) retry = 0 while True: try: with self.handle_abort(): self.nntp.body(id, *pos, **kw) break except failure_responses as err: [code, *msg] = err.response.split(maxsplit=1) if code == "400": [msg] = msg or (None,) if not msg: msg = "Server shut down connection" elif code[1] == "0" and not retry: msg = err.response else: raise self.log.write(msg + "\n") self.log_time() if retry >= 60: raise TimeoutError() self.close() time.sleep(retry) if not retry: start = time.monotonic() self.connect() if retry: retry *= 2 else: retry = time.monotonic() - start if retry <= 0: retry = 0.5 def group(self, *pos, **kw): with self.handle_abort(): return self.nntp.group(*pos, **kw) def over(self, *pos, **kw): with self.handle_abort(): return self.nntp.over(*pos, **kw) def hdr(self, *pos, **kw): with self.handle_abort(): return self.nntp.xhdr(*pos, **kw) @contextmanager def handle_abort(self): try: yield except failure_responses: raise # NNTP connection still intact except: # Protocol is disrupted so abort the connection straight away self.close() raise def close(self): if not self.nntp: return # Ignore failure of inappropriate QUIT command with suppress(NNTPError), self.nntp: pass self.nntp = None def log_time(self): # TODO: time duration formatter self.log.write("Connection lasted {:.0f}m\n".format((time.monotonic() - self.connect_time)/60))
#!/usr/bin/env python # -*- coding: UTF-8 *-* from nntplib import NNTP n = NNTP('your.nntp.server') rsp, ct, fst, lst, grp = n.group('comp.lang.python') rsp, anum, mid, data = n.article('110457') for eachLine in data: print eachLine From: "Alex Martelli" <alex@...> Subject: Re: Rounding Question Date: Wed, 21 Feb 2001 17:05:36 +0100 "Remco Gerlich" <remco@...> wrote: Jacob Kaplan-Moss <jacob@...> wrote in comp.lang.python: So I've got a number between 40 and 130 that I want to round up to the nearest 10. That is: 40 --> 40, 41 --> 50, ..., 49 --> 50, 50 --> 50, 51 --> 60 Rounding like this is the same as adding 5 to the number and then rounding down. Rounding down is substracting the remainder if you were to divide by 10, for which we use the % operator in Python. This will work if you use +9 in each case rather than +5 (note that he doesn't really want rounding -- he wants 41 to 'round' to 50, for ex). Alex >>> n.quit() '205 closing connection - goodbye!'
from nntplib import NNTP from time import strftime,time,localtime day = 24*60*60 window = 7 yesterday = localtime(time()-window*day) date = strftime('%y%m%d',yesterday) time = strftime('%H%M%S',yesterday) servername = 'news.gmane.org' # 18289 groupname = 'gmane.comp.python.apple' s = NNTP(servername) # group 'Return a tuple (response, count, first, last, name) where count is the (estimated) number of articles in the group, first is the first article number in the group, last is the last article number in the group, and name is the group name. The numbers are returned as strings.' resp,count,first,last,name = s.group(groupname) print('Group', name, 'has', count, 'articles, range', first, 'to', last) # over 'Return a pair (response, overviews). overviews is a list of (article_number, overview) tuples, one for each article selected by message_spec' resp,overviews = s.over((last-1,last)) for num,over in overviews: print(num)# 1-100 #print(over) # print(over.keys()) # ['xref', 'from', ':lines', ':bytes', 'references', 'date', 'message-id', 'subject'] print(over.get('date')) print(nntplib.decode_header(over.get('from'))) print(over.get('message-id')) print(over.get('subject'))
def newsgroup(G='', F='', C='', A=None, P=None, RESPONSE=None): """The article list for group G in framestyle F""" if G=='': return "Missing newsgroup name." group = G showframes = 1 show_articles = default_show_articles if os.environ.has_key('HTTP_USER_AGENT'): browser = os.environ['HTTP_USER_AGENT'] else: browser = "unknown" if string.find(browser, "Mozilla/") == 0: browser_version = string.atof(browser[8:string.index(browser, ' ')]) if browser_version >= 2.00: showframes = 3 if F != '': try: showframes = string.atoi(F) except AttributeError: showframes = 0 if C != '': try: show_articles = string.atoi(C) except AttributeError: show_articles = default_show_articles user = A acc_string = '' if A: acc_string = '&A=' + A password = P pass_string = '' if P: pass_string = '&P=' + P lines = [] RESPONSE.headers['expires'] = time.asctime(time.gmtime(time.time() + 60)) RESPONSE.write( """<HTML><HEAD><TITLE>Tokyo PC Users Group: %s</TITLE></HEAD>""" % group) try: try: news = NNTP(NEWS_SERVER) except: RESPONSE.write( "<BODY><B>Can not connect to server:</B> ", NEWS_SERVER) raise NewsError try: resp = news.shortcmd('MODE READER') except: RESPONSE.write( "<BODY><B>Can not communicate with server:</B> ", NEWS_SERVER) raise NewsError if user: resp = news.shortcmd('authinfo user '+user) if resp[:3] == '381': if not password: RESPONSE.write( "<BODY><B>Can not fetch newsgroup</B>") raise NewsError else: resp = news.shortcmd('authinfo pass '+password) if resp[:3] != '281': RESPONSE.write( "<BODY><B>Can not fetch newsgroup</B>") raise NewsError try: resp, count, first, last, name = news.group(group) except: RESPONSE.write( "<BODY><B>No such newsgroup:</B> " + group ) raise NewsError description = "" try: resp, lines = news.xgtitle(group) except: pass else: for line in lines: name, description = line if showframes == 0: RESPONSE.write( '<BODY BGCOLOR="#FFFFFF"><H1>%s</H1>' % group) RESPONSE.write( "<EM>%s</EM><P>" % cgi.escape(description)) elif showframes == 1 or showframes == 3: if description: description = "&D="+quote(description) RESPONSE.write( '<FRAMESET ROWS="33%,*">') RESPONSE.write( ' <FRAMESET COLS="220,*">') RESPONSE.write( ' <FRAME SRC="/cgi-bin/webnews/logo?G=%s%s" scrolling="auto">' % (group, description)) RESPONSE.write( ' <FRAME SRC="/cgi-bin/webnews/newsgroup?G=%s&F=2%s%s#last" scrolling="yes"> ' % (group, acc_string, pass_string)) RESPONSE.write( ' </FRAMESET>') if string.find(G, "ttalk") >= 0: RESPONSE.write( ' <FRAME SRC="http://ttalk.soholutions.com/welcome.html" scrolling="auto" name="d">') else: RESPONSE.write( ' <FRAME SRC="/webnews/welcome.html" scrolling="auto" name="d">') RESPONSE.write( '</FRAMESET><BODY BGCOLOR="#FFFFFF">') else: RESPONSE.write( '<BODY BGCOLOR="#FFFFFF">') if showframes == 3: raise NewsError if (show_articles > 0): ilast = string.atoi(last) ifirst = string.atoi(first) if ((ilast - ifirst + 1) > show_articles): first = "%d" % (ilast - show_articles + 1) RESPONSE.write( '<A HREF="/cgi-bin/webnews/newsgroup?G=%s&F=%d&C=0%s%s"><I>Retrieve earlier article headers</I></A> ' % (group, showframes, acc_string, pass_string)) try: resp, lines = news.xover(first, last) except: RESPONSE.write( "<B>Unable to get article list for:</B> " + group) raise NewsError RESPONSE.write( '<UL TYPE="none">') # pass 1: build a dictionary of message IDs ids = {} index = 0 for line in lines: art_nr, subject, poster, date, id, references, size, line_cnt = line ids[id] = index index = index + 1 # pass 2: discover child articles childof = [] subs = {} # subject_re_less = regex.symcomp("\([Rr]e:\)? *\(<real_subject>.*\)") subject_re_less = re.compile(r"(re:)?\s*(?P<real_subject>.*)") index = 0 for line in lines: art_nr, subject, poster, date, id, references, size, line_cnt = line childof.append(-1) # if subject_re_less.match(subject) > 0: # subject = subject_re_less.group('real_subject') srl = subject_re_less.match(subject) if srl: subject = srl.group('real_subject') # if there are references, use them (most recent first) if len(references) > 0: references.reverse() for ref in references: if ids.has_key(ref): childof[index] = ids[ref] break # if no references (or referee not found), use subject if childof[index] == -1: if subs.has_key(subject) : childof[index] = subs[subject] else: subs[subject] = index index = index + 1 # index = 0 # for line in lines: # art_nr, subject, poster, date, id, size, line_cnt, references = line # print index,childof[index],subject # index = index + 1 index = 0 for seq in childof: if seq == -1: show_article_and_kids(index, 0, lines, childof, acc_string, pass_string, RESPONSE) index = index + 1 # art_nr, subject, poster, date, id, size, line_cnt, references = line # name, email = parseaddr(poster) # print '<LI><A HREF="http:/cgi-bin/readnews.cgi?%s" TARGET="d">%s</A> (%s) %s, %s' % (quote(id), subject, line_cnt, name, time.strftime('%b %d, %H:%M', parsedate(date))) RESPONSE.write('<A NAME="last"> </A></UL>') finally: if showframes != 2: if string.find(G, "ttalk") >= 0: RESPONSE.write( """<P><HR><P>A service of the <A HREF="http://www.soholutions.com/">SoHolutions</A>.""") else: RESPONSE.write( """<P><HR><P>A service of the <A HREF="http://www.tpc.ml.org/">Tokyo PC Users Group</A>.""") # print "<P><ADDRESS>",os.environ['HTTP_USER_AGENT'],"</ADDRESS>" RESPONSE.write( """</BODY></HTML>""")
#__author: ZhengNengjin #__date: 2018/10/25 from nntplib import NNTP n = NNTP('your.nntp.server') rsp, ct, fst, lst, grp = n.group('comp,lang.python') rsp, anum, mid, data = n.article('110457') for eachLine in data: print(eachLine) n.quit()
listonly = 0 showhdrs = ['From', 'Subject', 'Date', 'Newsgroups', 'Lines'] try: import sys servername, groupname, showcount = sys.argv[1:] showcount = int(showcount) except: servername = 'news.rmi.net' groupname = 'comp.lang.python' # cmd line args or defaults showcount = 10 # show last showcount posts # connect to nntp server print 'Connecting to', servername, 'for', groupname from nntplib import NNTP connection = NNTP(servername) (reply, count, first, last, name) = connection.group(groupname) print '%s has %s articles: %s-%s' % (name, count, first, last) # get request headers only fetchfrom = str(int(last) - (showcount-1)) (reply, subjects) = connection.xhdr('subject', (fetchfrom + '-' + last)) # show headers, get message hdr+body for (id, subj) in subjects: # [-showcount:] if fetch all hdrs print 'Article %s [%s]' % (id, subj) if not listonly and raw_input('=> Display?') in ['y', 'Y']: reply, num, tid, list = connection.head(id) for line in list: for prefix in showhdrs: if line[:len(prefix)] == prefix: print line[:80]; break
# -*-coding:gbk-*- from nntplib import NNTP server = NNTP('news.foo.bar') print server.group('comp.lang.python.announce')[1]
from nntplib import NNTP server = NNTP('news.newsfan.net') server.group('comp.lang.python')[0]
# -*- coding:utf-8 -*- from nntplib import NNTP servername = 'web.aioe.org' group = 'comp.lang.python.announce' server = NNTP(servername) howmany = 10 resp, count, first, last, name = server.group(group) start = last - howmany + 1 resp, overviews = server.over((start, last)) for id, over in overviews: subject = over['subject'] resp, info = server.body(id) print('subject:', subject) print('-' * len(subject)) for line in info.lines: print(line.decode('latin1')) print() server.quit()
class Archive(object): @staticmethod def is_diff(body): return bool([line for line in body if line.startswith("diff ")]) def __init__(self, group, server): self.conn = NNTP(server) resp, count, first, last, name = self.conn.group(group) self.group = group self.server = server self.first = int(first) self.last = int(last) def get_number_from_user(self, msg_id): """ Convert something the user might input into a message id. These are: # An NNTP message number # A gmane link that includes the NNTP message number # The original Message-Id header of the message. NOTE: gmane's doesn't include the message number in STAT requests that involve only the Message-Id (hence the convolution of getting all the headers). """ msg_id = re.sub(r".*gmane.org/gmane.comp.version-control.git/([0-9]+).*", r"\1", str(msg_id)) _, n, id, result = self.conn.head(msg_id) for header in result: m = re.match(r"Xref: .*:([0-9]+)\s*$", header, re.I) if m: return int(m.group(1)) else: raise FatalError("No (or bad) Xref header for message '%s'" % msg_id) def get_patch_series(self, user_input, search_limit=100): """ Given an NNTP message number or a Message-Id header return an mbox containing the patches introduced by the author of that message. This handles the case where the threading is right *and* the patches are numbered in a simple scheme: [PATCH] this patch has no replies and stands on its own [PATCH 0/2] this is an introduction to the series |- [PATCH 1/2] the first commit |- [PATCH 2/2] the second commit [PATCH 1/3] this is the first commit |- [PATCH 2/3] and this is the second |- [PATCH 3/3] and this is the third TODO: it would be nice to make the search more efficient, we can use the numbers in [PATCH <foo>/<bar>] to stop early. """ start_id = self.get_number_from_user(user_input) messages = limit(self.messages_starting_from(start_id), search_limit) try: thread = Thread(messages.next()) except StopIteration: raise FatalError("No message at id '%s' using XOVER") n_since_last = 0 for message in messages: if n_since_last > 5: break elif thread.should_include(message): n_since_last = 0 thread.append(message) else: n_since_last += 1 else: raise FatalError('did not find end of series within %s messages', search_limit) for message in self.xover(start_id - 5, start_id -1): if thread.should_include(message): thread.append(message) return self.mboxify(thread) def mboxify(self, thread): """ Convert a thread into an mbox for application via git-am. """ lines = [] for message in thread.in_order(): _, number, msg_id, body = self.conn.body(str(message.number)) # git-am doesn't like empty patches very much, and the 0/X'th patch is # often not a patch, we skip it here. (TODO, warn the user about this) if re.search(r" 0+/[0-9]+", message.subject) and not self.is_diff(body): continue poster = parseaddr(message.poster)[0] date = ctime(mktime(parsedate(message.date))) lines.append("From %s %s" % (poster, date)) lines.append("From: %s" % message.poster) lines.append("Subject: %s" % message.subject) lines.append("Date: %s" % message.date) lines.append("Message-Id: %s" % message.msg_id) lines.append("Xref: %s %s:%s" % (self.server, self.group, message.number)) lines.append("References: %s" % "\n\t".join(message.references)) lines.append("") lines += body lines.append("") return "\n".join(lines) def messages_starting_from(self, start_id): """ Generate all message headers starting from the given id and working upwards. """ while start_id < self.last: next_id = min(start_id + 20, self.last) for message in self.xover(start_id, next_id): yield message start_id = next_id + 1 def xover(self, begin, end): """ Get the headers for the messages with numbers between begin and end. """ if begin == end: return [] _, result = self.conn.xover(str(min(begin, end)), str(max(begin, end))) result = [Message(int(number), subject, poster, date, msg_id, references) for (number, subject, poster, date, msg_id, references, size, lines) in result] return sorted(result, key=lambda x: x.number)
# Open file to write f = open('samplegroup_fra.xml', 'w') f.write('<dialog>\n') f.close() # Open file for statistics g = open('stats_message.csv', 'wb') group_writer = csv.writer(g) group_writer.writerow(['Group', 'No. of messages', 'Mean authors', 'Max authors']) g.close() # Iterate through French groups (only subset at present) for group in french_groups[]: # Read number of messages in group resp, count, first, last, name = s.group(group) print "Group [" + group + "] has " + count + " articles (" + first + ", " + last + ")" # Skip empty newsgroups if count > 0: # Read items info from group print "- Reading items" # DEBUG - ******** THIS NEEDS TO BE REMOVED, IT JUST LOOKS AT LAST 50 MESSAGES TO SAVE TIME FOR NOW ********* #if int(last)-int(first) > 200: # first = str(int(last)-200) # print "-- DEBUG: Truncating to (" + first + "," + last + ")" # DEBUG resp, items = s.xover(first, last)
class DanskGruppenArchive(object): """Class that provides an interface to Dansk-gruppens emails archive on gmane """ def __init__(self, article_cache_size=300, cache_file=None): """Initialize local variables""" # Connect to news.gmane.org self.nntp = NNTP('news.gmane.org') # Setting the group returns information, which right now we ignore self.nntp.group('gmane.comp.internationalization.dansk') # Keep a local cache in an OrderedDict, transferred across session # in a pickled version in a file self.article_cache_size = article_cache_size self.cache_file = cache_file if cache_file and path.isfile(cache_file): with open(cache_file, 'rb') as file_: self.article_cache = pickle.load(file_) logging.info('Loaded %i items from file cache', len(self.article_cache)) else: self.article_cache = OrderedDict() def close(self): """Quit the NNTP session and save the cache""" self.nntp.quit() if self.cache_file: with open(self.cache_file, 'wb') as file_: pickle.dump(self.article_cache, file_) logging.info('Wrote %i items to cache file', len(self.article_cache)) @property def last(self): """Return the last NNTP ID as an int""" return self.nntp.group('gmane.comp.internationalization.dansk')[3] def _get_article(self, message_id): """Get an article (cached) Args: message_id (int): The NNTP ID of the message Returns: list: List of byte strings in the message """ # Clear excess cache if len(self.article_cache) > self.article_cache_size: self.article_cache.popitem(last=False) # Check if article is in cache and if not, put it there if message_id not in self.article_cache: # nntp.article() returns: response, information # pylint: disable=unbalanced-tuple-unpacking _, info = self.nntp.article(message_id) self.article_cache[message_id] = info return self.article_cache[message_id] @staticmethod def _article_to_email(article): """Convert a raw article to an email object Args: article (namedtuple): An article named tuple as returned by NNTP Returns: email.message: An email message object """ # article lines are a list of byte strings decoded_lines = [line.decode('ascii') for line in article.lines] article_string = '\n'.join(decoded_lines) # Make an email object return email.message_from_string(article_string) def get_subject(self, message_id): """Get the subject of an message Args: message_id (int): The NNTP ID of the the message Returns: str: The subject of the article """ article = self._get_article(message_id) mail = self._article_to_email(article) # The subject may be encoded by NNTP, so decode it return decode_header(mail['Subject']) def get_body(self, message_id): """Get the body of a message Args: message_id (int): The NNTP ID of the the message Returns: str: The body of the article as a str or None if no body could be found or succesfully decoded """ article = self._get_article(message_id) mail = self._article_to_email(article) # Walk parts of the email and look for text/plain content type for part in mail.walk(): if part.get_content_type() == 'text/plain': body = part.get_payload(decode=True) # Find the text encoding from lines like: # text/plain; charset=UTF-8 # text/plain; charset=utf-8; format=flowed # Encoding sometimes has "" around it, decode is OK with that for type_part in part['Content-Type'].split(';'): if type_part.strip().startswith('charset='): encoding = type_part.replace('charset=', '') break else: message = 'Looking for the character encoding in the '\ 'string "%s" went wrong' logging.warning(message, part['Content-Type']) return None # Decode and return the body try: body = body.decode(encoding) except LookupError: message = 'Do not know how to handle a body with '\ 'charset: %s' logging.warning(message, encoding) return None return body def get_attachment(self, message_id, filename): """Get attachment by filename Args: message_id (int): The NNTP ID of the the message filename (str): The filename for the attachment Returns: bytes: The binary content of the attachment """ return self.get_attachments(message_id).get(filename) def get_attachments(self, message_id): """Get attachments Args: message_id (int): The NNTP ID of the the message Returns: dict: Dict with attachments where keys are filenames and values are their binary content """ article = self._get_article(message_id) mail = self._article_to_email(article) attachments = {} # Walk parts of the email and look for application/octet-stream # content type for part in mail.walk(): content_disp = part['Content-Disposition'] if not (content_disp and content_disp.startswith('attachment')): continue # Get the filename from a line like: Content-Disposition: # attachment; filename="hitori.master.da.podiff" filename = None for disp_part in content_disp.split(';'): if disp_part.strip().startswith('filename='): filename = disp_part.strip().replace('filename=', '') # Strip " from filename filename = filename.strip('"') if filename is None: message = 'Unable to extract filename from '\ 'Content-Disposition: %s' logging.warning(message, part['Content-Disposition']) raise Exception('Unable to extract filename') attachments[filename] = part.get_payload(decode=True) return attachments
import nntplib from nntplib import NNTP s = NNTP('news.gmane.org') resp, count, first, last, name = s.group('gmane.comp.python.committers') print ('Group', name, 'has', count, 'articles, range', first, 'to', last) resp, subs = s.xhdr('subject',first) for id, sub in subs[-10:]: print (id, sub) try : for id, sub in subs[-10:]: print (id, sub) except TypeError as e: print(e) print(s.quit()) # NNTP.
import textwrap import re # print strftime('%y%m%d') # start = localtime(time()-60*60*24*365) # print strftime('%y%m%d',start) class SimpleWebSource: def __init__(self, url, titlePattern, bodyPattern): self.url = url self.titlePattern = re.compile(titlePattern) self.bodyPattern = re.compile(bodyPattern) def getItems(self): text = urlopen(self.url).read() titles = self.titlePattern.findall(text) bodies = self.bodyPattern.findall(text) for title in titles: print title[1].decode('utf-8') print bodies bbc_url = 'http://news.sina.com.cn' bbc_title = r'<a target="_blank" href="(.*?)">(.*?)</a>' bbc_body = r'(?s)</a>\s*<br />\s*(.*?)\s*<' # bbc = SimpleWebSource(bbc_url,bbc_title,bbc_body) # bbc.getItems() server = NNTP('129.69.1.59') server.group('camp.lang.python')[0]
class DownloadSpots(object): def __init__(self): try: self.news = NNTP(NEWS_SERVER, NNTP_PORT, NNTP_USERNAME, NNTP_PASSWORD) except NNTPTemporaryError as e: raise SpotError('NNTP', e) except socket.error as e: raise SpotError('Connection', e) self.conn = sqlite3.connect(NEWS_SERVER + '.db') self.conn.row_factory = sqlite3.Row self.cur = self.conn.cursor() self.cur.executescript('''\ PRAGMA synchronous = OFF; PRAGMA journal_mode = MEMORY; PRAGMA temp_store = MEMORY; PRAGMA count_changes = OFF; ''') def __del__(self): print 'quit!' if hasattr(self, 'news'): self.news.quit() if hasattr(self, 'cur'): self.cur.close() if hasattr(self, 'conn'): self.conn.close() def make_table(self): sql = '''\ CREATE TABLE IF NOT EXISTS spots ( id int PRIMARY KEY, full_id str, cat int, title str, poster str, date int, size int, erotiek int, subcats str, modulus int, keyid int, c_count int ); CREATE TABLE IF NOT EXISTS comments ( id int PRIMARY KEY, full_id str, spot_full_id str ); CREATE INDEX IF NOT EXISTS spots_full_id_index on spots(full_id); ''' self.cur.executescript(sql) self.conn.commit() def download_detail(self, id): '''Get information about a spot. Args: id (int/string): the nntp id of the spot. Can be: - a short id: 123456 - a long id: '*****@*****.**' Returns: a dict with the following keys: - nzb: a list of nntp id's of the nzb file - image: the location of the image file: url or nntp id - website: url of the website - desc: a description of the spot - title2: the title of the spot (is called title2 to avoid a conflict) ''' id = str(id) self.news.group('free.pt') print id head = self.news.head(id)[-1] xmltext = ''.join(item[7:] for item in head if item.startswith('X-XML:')) xml = etree.XML(xmltext) xmldict = defaultdict(list) xmldict['nzb'] = [i.text for i in xml.find('.//NZB').iter('Segment')] imgfind = xml.find('.//Image') if imgfind is not None: if imgfind.find('.//Segment') is not None: xmldict['image'] = imgfind.find('.//Segment').text else: xmldict['image'] = imgfind.text else: xmldict['image'] = None webfind = xml.find('.//Website') if webfind is not None: xmldict['website'] = webfind.text else: xmldict['website'] = None xmldict['desc'] = bb2html('\n'.join(self.news.body(id)[-1])) xmldict['title2'] = xml.find('.//Title').text print xmldict return xmldict def download_image(self, article, imgnr): '''Download and save an image file. Args: article: Location of the file, can be: - None - an url: 'http://url.to.image.ext' - a nntp id: '*****@*****.**' imgnr: filename of the saved image ''' print imgnr file = 'temp/%s.picture' % imgnr if article is None: shutil.copy('none.png', file) elif article.startswith('http'): if 'imgchili.com' in article: article = re.sub(r't([0-9]\.imgchili)', r'i\1', article) try: print urllib.urlretrieve(article, file) except: print 'Image download error' shutil.copy('none.png', file) elif '@' in article: article = '<%s>' % article print article try: self.news.group('alt.binaries.ftd') data = ''.join(self.news.body(article)[-1]) except NNTPTemporaryError as e: shutil.copy('none.png', file) raise SpotError('NNTP', e) else: data = data.replace('=A', '\x00').replace('=B', '\r') \ .replace('=C', '\n').replace('=D', '=') with open(file, 'wb') as f: f.write(data) else: shutil.copy('none.png', file) def download_nzb(self, articles, title): '''Download and save a nzb file. Args: articles: a list of nntp id's: ['*****@*****.**'] title (string): the filename of the saved nzb file (must be already escaped) ''' print 'download_nzb' file = 'temp/%s.nzb' % title self.news.group('alt.binaries.ftd') print articles data = ''.join(''.join(self.news.body('<%s>' % article)[-1]) for article in articles) data = data.replace('=A', '\x00').replace('=B', '\r') \ .replace('=C', '\n').replace('=D', '=') data = zlib.decompress(data, -zlib.MAX_WBITS) with open(file, 'wb') as f: f.write(data) def update_spots(self): '''Download new spots and save them to the database. Yields: total: total amount of spots to be downloaded subtotal: amount of spots already done ''' print 'Opening database...', self.make_table() self.cur.execute('SELECT count(id) FROM spots') oude_lengte = self.cur.fetchone()[0] print 'Done' print oude_lengte, 'spots in database' print 'Connecting...', last = int(self.news.group('free.pt')[3]) print 'Done' if oude_lengte: self.cur.execute('SELECT max(id) FROM spots') current = self.cur.fetchone()[0] else: current = last - (NR_OF_SPOTS * 100) delta = 2000 total = last - current print 'Current:', current print 'Last:', last print total, 'spots op te halen' yield total if current >= last: print 'Geen nieuwe spots!' yield 'end' return print 'Getting new spots...', self.cur.execute('DROP INDEX IF EXISTS spots_full_id_index') self.conn.commit() for i in xrange(0, total, delta): for (id, subject), (id, fromline), (id, full_id) in \ zip(*(self.news.xhdr(header, '%s-%s' % (current + 1 + i, current + delta + i))[-1] for header in ('subject', 'from', 'Message-ID'))): subject = re.sub('^[\t]', '', subject).split('\t')[0] if subject[:7] == 'DISPOSE': continue subject = subject.decode('latin-1') fromline = fromline.decode('latin-1') subject = re.sub('\s\|\s\w+$', '', subject) poster = re.sub('(\t[^\t]+\t)?([^\s]+)\s<.+$', r'\2', fromline) try: info = re.findall(r'<[^>]+>', fromline)[0].split('@') modulus = info[0] info = info[1].split('.')[:4] date = info[3] except IndexError: continue if info[0][1] == '7': modulus = modulus.split('.')[0][1:] else: modulus = 0 subcats = '|'.join(info[0][2:][x:x + 3] for x in xrange(0, len(info[0][2:]), 3)) erotiek = 0 if info[0][0] == '1': for genre in re.findall('d[0-9]{2}', subcats): if ((23 <= int(genre[1:]) <= 26) or (72 <= int(genre[1:]) <= 89)): erotiek = 1 break sql = ('INSERT INTO spots (id, full_id, cat, title, poster, date, ' 'size, erotiek, subcats, modulus, keyid, c_count) ' 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)') self.cur.execute(sql, (id, full_id, info[0][0], subject, poster, date, info[1], erotiek, subcats, modulus, info[0][1])) self.conn.commit() yield int(id) - current # subtotal self.cur.execute('CREATE INDEX spots_full_id_index on spots(full_id)') self.conn.commit() print 'Spots klaar!' self.cur.execute('SELECT count(id) FROM spots') lengte = self.cur.fetchone()[0] print lengte - oude_lengte, 'nieuwe spots!' self.news.quit() yield 'end' def update_comments(self): '''Download new comments and save them to the database. Yields: total: total amount of comments to be downloaded subtotal: amount of comments already done ''' print 'comments' print 'Opening database...', self.make_table() self.cur.execute('SELECT count(id) FROM comments') oude_lengte = self.cur.fetchone()[0] print 'Done' print oude_lengte, 'comments in database' print 'Connecting...', last = int(self.news.group('free.usenet')[3]) print 'Done' if oude_lengte: self.cur.execute('SELECT max(id) FROM comments') current = self.cur.fetchone()[0] print current else: current = last - (NR_OF_SPOTS * 500) delta = 2000 total = last - current print 'Current:', current print 'Last:', last print total, 'comments op te halen' yield total if current >= last: print 'Geen nieuwe comments!' yield 'end' return print 'begin reacties', time.time() for i in xrange(0, total, delta): for (id, ref), (id, msgid) in \ zip(*(self.news.xhdr(header, '%s-%s' % (current + 1 + i, current + delta + i))[-1] for header in ('References', 'Message-ID'))): self.cur.execute('INSERT INTO comments VALUES (?, ?, ?)', (id, msgid, ref)) self.cur.execute('UPDATE spots SET c_count = c_count + 1 WHERE full_id = ?', (ref,)) yield int(id) - current # subtotal self.conn.commit() print 'Reacties klaar' self.cur.execute('SELECT count(id) FROM comments') lengte = self.cur.fetchone()[0] print lengte - oude_lengte, 'nieuwe comments!' self.news.quit() yield 'end' def show(self, search_col, op, search, search_rest='', limit=NR_OF_SPOTS, columns='*'): '''Perform a sql query. Args: search_col: the column to use in the where clause op: the operator to use in the where clause search: the search value to use in the where clause search_rest: additional conditions for the where clause limit: the number of rows to select columns: the columns to select Returns: a list of tuples with the results of the sql query ''' self.make_table() sql = '''\ SELECT %s FROM spots WHERE %s %s :search %s ORDER BY id DESC LIMIT %s ''' % (columns, search_col, op, search_rest, limit) results = map(Spot, self.cur.execute(sql, (search,))) print len(results) return results
__author__ = 'sks' from nntplib import NNTP from time import strftime, time, localtime day = 24 * 60 * 60 yesterday = localtime(time() - day) date = strftime('%y%m%d', yesterday) hour = strftime('%H%M%S', yesterday) servername = 'data.eastmoney.com' group = 'comp.lang.python.announce' server = NNTP(servername) print(server.group(group)[0]) # to be continued #ids = server.newnews()
#!/usr/local/bin/python3 from nntplib import NNTP server = NNTP('news.aioe.org') resp, count, first, last, name = server.group('sci.logic') print('Group', name, 'has', count, 'articles, range', first, 'to', last) resp, subs = server.xhdr('subject', str(first) + '-' + str(last)) for id, text in subs[-10:]: print(id, text) server.quit()
import sqlite3 from nntplib import NNTP group = 'alt.binaries.movies.x264' chunk_size = 100000 sql = sqlite3.connect('usenet.db') print "About to start dumping articles from " + group serv = NNTP('news.astraweb.com', 119, 'arealmuto', 'stock1114') resp = serv.group(group) count = int(resp[1]) first = int(resp[2]) last = int(resp[3]) print "There are " + str(count) + " articles to get" print "First: " + str(first) print "Last: " + str(last) print "Using chunks size of: " + str(chunk_size) print "It should take " + str(count/chunk_size) + " requests to finish" id = int(first) i = 0 master_list = [] while id < last: print str(i) +": Getting id's " + str(id) + " - " + str(id + chunk_size) resp, list = serv.xover(str(id), str(id+ chunk_size)) print "Done fetching" print "Adding to master list" for line in list:
#/usr/bin/python # *-*coding:utf-8 *-* #python NNTP # group组返回响应信息 from nntplib import NNTP with NNTP('news.gmane.org') as n: print(n.group('gmane.comp.python.committers')) server = NNTP('web.aioe.org') print(server.group('comp.lang.python.announce')[0]) #('211 4419 1 4419 gmane.comp.python.committers', 4419, 1, 4419, 'gmane.comp.python.committers') #211 100 3923 4023 comp.lang.python.announce ''' ('211 4371 1 4371 gmane.comp.python.committers', 4371, 1, 4371, 'gmane.comp.python.committers') 411 开头意味着服务器没有这个组 nntplib.NNTPTemporaryError: 411 No such group comp.lang.python.announce 211开头 基本上意味着服务器拥有你所请求的组 ('211 98 3911 4008 comp.lang.python.announce', 98, 3911, 4008, 'comp.lang.python.announce') 还有可能得到一个带有类似错误信息的的异常 如果引发了异常 可能是服务器的名字写错了 另外一种可能是在创建服务器对象和调用group方法之间 '超时了', 服务器允许用户保持连接的时间很短(比如10秒) 如果输入速度不够快 把代码放到脚本里, 或着将服务器对象的创建和方法的调用放在同一行内 (以分号隔开)
from nntplib import NNTP servername = 'news.foo.bar' group = 'comp.lang.python.announce' server = NNTP(servername) howmany = 10 resp, count, first, last, name = server.group(group) start = last - howmany + 1 resp, overviews = server.over((start, last)) for id, over in overviews: subject = over['subject'] resp, info = server.body(id) print(subject) print('-' * len(subject)) for line in info.lines: print(line.decode('latin1')) print() server.quit()
listonly = 0 showhdrs = ['From', 'Subject', 'Date', 'Newsgroups', 'Lines'] try: import sys servername, groupname, showcount = sys.argv[1:] showcount = int(showcount) except: servername = 'news.gmane.org' groupname = 'gmane.comp.python.general' # cmd line args or defaults showcount = 10 # show last showcount posts # connect to nntp server print 'Connecting to', servername, 'for', groupname from nntplib import NNTP connection = NNTP(servername) (reply, count, first, last, name) = connection.group(groupname) print '%s has %s articles: %s-%s' % (name, count, first, last) # get request headers only fetchfrom = str(int(last) - (showcount-1)) (reply, subjects) = connection.xhdr('subject', (fetchfrom + '-' + last)) # show headers, get message hdr+body for (id, subj) in subjects: # [-showcount:] if fetch all hdrs print 'Article %s [%s]' % (id, subj) if not listonly and raw_input('=> Display?') in ['y', 'Y']: reply, num, tid, list = connection.head(id) for line in list: for prefix in showhdrs: if line[:len(prefix)] == prefix: print line[:80]; break
from nntplib import NNTP #server = NNTP('news.easynews.com') #server = NNTP('news.giganews.com') #server = NNTP('news.mozilla.org') server = NNTP('news.kornet.net') print server.group('comp.lang.python.announce')[0] """ | group(self, name) | Process a GROUP command. Argument: | - group: the group name | Returns: | - resp: server response if successful | - count: number of articles (string) | - first: first article number (string) | - last: last article number (string) | - name: the group name """ group = server.group('han.test') print repr(group) first = group[2] last = group[3] print "first ", first print "last ", last i = 0 for id in range(int(first), int(last)): i += 1 print(server.article(str(id)))
class Client(Context): def __init__(self, log, hostname, port=None, username=None, password=None, *, debuglevel=None, **timeout): self.log = log self.hostname = hostname self.port = port self.username = username self.password = password self.debuglevel = debuglevel self.timeout = timeout Context.__init__(self) self.connect() def connect(self): address = net.format_addr((self.hostname, self.port)) self.log.write("Connecting to {}\n".format(address)) if self.port is None: port = () else: port = (self.port, ) self.connect_time = time.monotonic() self.nntp = NNTP(self.hostname, *port, **self.timeout) with ExitStack() as cleanup: cleanup.push(self) if self.debuglevel is not None: self.nntp.set_debuglevel(self.debuglevel) self.log.write("{}\n".format(self.nntp.getwelcome())) if self.username is not None: self.log.write("Logging in as {}\n".format(self.username)) with self.handle_abort(): self.nntp.login(self.username, self.password) self.log.write("Logged in\n") cleanup.pop_all() def body(self, id, *pos, **kw): id = "<{}>".format(id) retry = 0 while True: try: with self.handle_abort(): self.nntp.body(id, *pos, **kw) break except failure_responses as err: [code, *msg] = err.response.split(maxsplit=1) if code == "400": [msg] = msg or (None, ) if not msg: msg = "Server shut down connection" elif code[1] == "0" and not retry: msg = err.response else: raise self.log.write(msg + "\n") self.log_time() if retry >= 60: raise TimeoutError() self.close() time.sleep(retry) if not retry: start = time.monotonic() self.connect() if retry: retry *= 2 else: retry = time.monotonic() - start if retry <= 0: retry = 0.5 def group(self, *pos, **kw): with self.handle_abort(): return self.nntp.group(*pos, **kw) def over(self, *pos, **kw): with self.handle_abort(): return self.nntp.over(*pos, **kw) def hdr(self, *pos, **kw): with self.handle_abort(): return self.nntp.xhdr(*pos, **kw) @contextmanager def handle_abort(self): try: yield except failure_responses: raise # NNTP connection still intact except: # Protocol is disrupted so abort the connection straight away self.close() raise def close(self): if not self.nntp: return # Ignore failure of inappropriate QUIT command with suppress(NNTPError), self.nntp: pass self.nntp = None def log_time(self): # TODO: time duration formatter self.log.write("Connection lasted {:.0f}m\n".format( (time.monotonic() - self.connect_time) / 60))
class NewsWatcher(MessageWatcher): def __init__(self, server, groups, user=None, pw=None, port=None, tag=None): MessageWatcher.__init__(self) self.server = server self.groups = groups self.nntp = None # the NNTP connection object self.user = user self.pw = pw self.port = port self.tag = tag self.last = {} self.timeout = None self.pollInterval = 60 self.debug = 0 def __repr__(self): return "<NewsWatcher %s:%s (%s)>" % (self.server, self.port, ",".join(self.groups)) def __getstate__(self): d = MessageWatcher.__getstate__(self) d['nntp'] = None # just in case return d def start(self): port = self.port if not port: port = NNTP_PORT self.nntp = NNTP(self.server, port, self.user, self.pw, readermode=1) # only look for messages that appear after we start. Usenet is big. if not self.last: # only do this the first time for g in self.groups: resp, count, first, last, name = self.nntp.group(g) self.last[g] = int(last) if self.debug: print "last[%s]: %d" % (g, self.last[g]) self.timeout = gtk.timeout_add(self.pollInterval*1000, self.doTimeout) def stop(self): self.nntp.quit() self.nntp = None if self.timeout: gtk.timeout_remove(self.timeout) self.timeout = None def doTimeout(self): self.poll() return gtk.TRUE # keep going def poll(self): #print "polling", self for g in self.groups: resp, count, first, last, name = self.nntp.group(g) for num in range(self.last[g]+1, int(last)+1): try: resp, num, id, lines = self.nntp.article("%d" % num) except NNTPError: continue name = "%s:%d" % (g, int(num)) if self.debug: print "got", name if self.tag: if not filter(lambda line, tag=tag: line.find(tag) != -1, lines): continue self.parseMessage(name, name, time.time(), lines) self.last[g] = int(last)
from nntplib import NNTP server = NNTP('news2.neva.ru') print server.group('alt.sex.telephone')