#coding=utf-8
#
from nntplib import NNTP
from time import strftime, time, localtime
day = 24 * 60 * 60 # Number of seconds in one day
yesterday = localtime(time() - day)
date = strftime('%y%m%d', yesterday)
hour = strftime('%H%M%S', yesterday)
servername = 'news.mixmin.net'
group = 'talk.euthanasia'
server = NNTP(servername)
ids = server.newnews(group, date, hour)[1]
for id in ids:
head = server.head(id)[3]
for line in head:
if line.lower().startswith('subject:'):
subject = line[9:]
break
body = server.body(id)[3]
print subject
print '-'*len(subject)
print '\n'.join(body)
server.quit()
def getItems(self):
start = localtime(time() - self.window * day)
date = strftime('%y%m%d', start)
hour = strftime('%H%M%S', start)
full_date = date + time
server = NNTP(self.servername)
ids = server.newnews(
self.group, datetime.datetime.strptime(full_date,
'%y%m%d%H%M%S'))[1]
for id in ids:
lines = server.article(id)[3]
message = message_from_string('\n'.join(lines))
title = message['subject']
body = message.get_payload()
if message.is_multipart():
body = body[0]
yield NewsItem(title, body)
server.quit()
def getItems(self):
server = NNTP(self.servername)
group = server.group(self.group)
first = group[2]
last = str(int(group[2]) + 10)
#last = str(int(group[2]) + 2)
for id in range(int(first), int(last)):
lines = server.article(str(id))[3]
message = message_from_string('\n'.join(lines))
title = message['subject']
#print(repr(dir(message)))
print(message.get_content_type())
if 'text' in message.get_content_type():
body = message.get_payload()
else:
body = 'Body is not text'
if 'ybegin' in message.get_payload():
body = 'hmm'
if message.is_multipart():
#body = body[0]
body = 'Multi part is not implemented yet'
yield NewsItem(title, body)
server.quit()
class NNTPSource(Source):
PREFIX = 'dispose'
def __init__(self, serverName, group, howmany):
super().__init__()
self.serverName = serverName
self.type = Source.NNTP
self.group = group
self.howmany = howmany
self.server = NNTP(self.serverName)
def getNewsItem(self):
resp, count, first, last, name = self.server.group(self.group)
resp = resp.split(' ')[0]
if resp == '211': # 正常响应
start = last - self.howmany + 1
resp, overviews = self.server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = self.server.body(id)
body = '\n'.join(
line.decode('latin')
for line in info.lines) + '\n\n' # 使用生成器推导,转字符串
yield NewsItem(title, body, self.NNTP)
else:
yield None
self.server.quit()
def getDisposeName(self):
return self.PREFIX + self.type
pass
def disposeNNTP(self, news):
print('NNTP新闻生产完毕')
pass
def get_items(self): #新闻生成器
server = NNTP(self.servername)
resp, count, first, last, name = server.group(self.group) #新闻组信息列表
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = server.body(id)
body = '\n'.join(line.decode() for line in info.lines) + '\n\n'
yield NewsItem(title, body)
server.quit()
def main():
s = NNTP(settings.nntp_hostname, settings.nntp_port)
resp, groups = s.list()
# check all of the groups, just in case
for group_name, last, first, flag in groups:
resp, count, first, last, name = s.group(group_name)
print "\nGroup", group_name, 'has', count, 'articles, range', first, 'to', last
resp, subs = s.xhdr('subject', first + '-' + last)
for id, sub in subs[-10:]:
print id, sub
s.quit()
def use_NNTP():
for server in NNTP_server_list:
print server
server = raw_input('Which NNTP_server do you want to go?\n')
while True:
if server not in NNTP_server_list:
print 'invalid srever name,try again!'
server = raw_input()
else:
break
s = NNTP(server)
print "s = NNTP('news.newsfan.net') is OK!"
(resp, lst) = s.list()
while True:
for i, elem in enumerate(lst):
print i, elem[0].decode('gbk')
num = raw_input('Which group do you want to go?\n')
try:
rsp, ct, first, last, grp = s.group(lst[int(num)][0])
print "Article's range is:%s to %s." % (first, last)
(resp, subs) = s.xhdr('subject', (str(first) + '-' + str(last)))
except:
print format_exc()
print 'invalid input!try again!!!'
sleep(3)
continue
for subject in subs:
try:
print subject[0], subject[1].decode('gbk')
except:
print subject[0], subject[1]
while True:
try:
number = raw_input('Which article do you want to read?\n')
if number == 'q':
break
f = open("NNTPfile", 'w')
(reply, num, id, list) = s.body(str(number), f)
f = open("NNTPfile", 'r')
for eachLine in f:
try:
print eachLine.decode('gbk'),
except:
print eachLine
f.close()
print 'Press any to continue...(Press q to return...)'
if raw_input() == 'q':
break
except:
print format_exc()
print 'invalid input!try again!!!'
sleep(3)
s.quit()
return
def main():
s = NNTP(settings.nntp_hostname, settings.nntp_port)
resp, groups = s.list()
# check all of the groups, just in case
for group_name, last, first, flag in groups:
resp, count, first, last, name = s.group(group_name)
print "\nGroup", group_name, 'has', count, 'articles, range', first, 'to', last
resp, subs = s.xhdr('subject', first + '-' + last)
for id, sub in subs[-10:]:
print id, sub
s.quit()
def get_items(self):
server = NNTP(self.servername)
resp, count, first, last, name = server.group(self.group)
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over["subject"])
resp, info = server.body(id)
body = "\n".join(line.decode("latin")
for line in info.lines) + "\n\n"
yield NewsItem(title, body)
server.quit()
def get_items(self):
server = NNTP(self.servername)
resp, count, first, last, name = server.group(self.group)
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = server.body(id)
body = '\n'.join(line.decode('latin')
for line in info.lines) + '\n\n'
yield NewsItem(title, body)
server.quit()
def get_items(self):
server = NNTP(self.servername)
_, count, first, last, name = server.group(self.group)
start = last - self.how_many + 1
_, overviews = server.over((start, last))
for ID, over in overviews:
title = decode_header(over['subject'])
_, info = server.body(ID)
body = '\n'.join(line.decode('latin') for line in info.lines)
yield NewsItem(title, body)
server.quit()
def get_items(self):
server = NNTP(self.servername)
#服务器响应、新闻组包含的消息数、第一条和最后一条消息编号、新闻组名称
resp, count, first, last, name = server.group(self.group)
#确定要获取的文章编号区间的起始位置
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = server.body(id)
body = '\n'.join(line.decode('latin')
for line in info.lines) + '\n\n'
yield NewsItem(title, body)
server.quit()
def getItems(self):
# yesterday = localtime(time()-self.window*day)
# date = strftime('%y%m%d',yesterday)
# time = strftime('%H%M%S',yesterday)
# create a nntp server
s = NNTP(self.servername)
resp,count,first,last,name = s.group(self.groupname)
resp,overviews = s.over((last-1,last))
for num,over in overviews:
title = over.get('subject')
resp,body = s.body(num)
# create a generator to iterate news
if title and body:
yield NewsItem(title,body)
s.quit()
def getItems(self):
server = NNTP(self.servername)
(resp, count, first, last, name) = server.group(self.group)
(resp, subs) = server.xhdr('subject', (str(first) + '-' + str(last)))
for subject in subs[-10:]:
title = subject[1]
(reply, (num, id, list)) = server.body(subject[0])
# list是一个列表，但是是bytes编码的，需要把每一个元素都解码成string
body = []
#print(list)
for l in list:
body.append(l.decode('gbk')) # 注意，这里用utf-8解码会出现解码错误
#print(body)
body = ''.join(body)
yield NewsItem(title, body)
server.quit()
def add_new_jobs(group_name):
db = Connection().usenet
max_h = db.control.find({"group": group_name}).sort('end', DESCENDING).limit(1)[0]['end'] + 1
if not max_h: max_h = 0
news = NNTP('eu.news.astraweb.com', user='******', password='******')
group = dict_group(news, group_name)
news.quit()
i = max(group['first'], max_h)
if max_h > group['last']: return
while(i+100000 < group['last']):
db.control.insert({'init': i, 'end': i+99999, 'done':False, "group": group_name})
i += 100000
db.control.insert({'init': i, 'end': group['last'], 'done':False, "group": group_name})
def getItems(self):
server = NNTP(self.servername)
(resp, count, frist, last, name) = server.group(self.group)
(resp, subs) = server.xhdr('subject', (str(frist) + '-' + (last)))
for subject in subs[-10:]:
title = subject[1]
(reply, num, id, list) = server.body(subject[0])
body = ''.join(list)
#print(num) #186919
#print(title) #Re: Find out which module a class came from
#print(''.join(list))#prano wrote:> But for merely ordinary obfuscation caused by poor...
yield NewsItem(title, body)
server.quit()
def get_items(self):
for servername in KNOWN_NNTP_SERVERS:
try:
server = NNTP(servername)
resp, count, first, last, name = server.group(self.group)
start = last - self.howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
title = decode_header(over['subject'])
resp, info = server.body(id)
body = '\n'.join(
line.decode('latin1') for line in info.lines) + '\n\n'
yield NewsItem(title, body, "NNTP NewsGroup " + self.group)
server.quit()
break
except:
continue
return []
def getItems(self):
start = localtime(time()-self.window*day)
date = strftime('%y%m%d',start)
hour = strftime('%H%M%S',start)
server = NNTP(self.servername)
ids = server.newnews(self.group,date,hour)[1]
for id in ids:
lines = server.article(id)[3]
message = message_from_string('\n'.join(lines))
title = memssage['subject']
body = message.get_payload()
if message.is_multipart():
body = body[0]
yield NewsItem(title,body)
server.quit()
def getItems(self):
server = NNTP(self.servername)
_, count, first, last, name = server.group(self.group)
_, subs = server.xhdr(
'subject', (str(first) + '-' + str(last)))
for sub in subs[:10]: # default last ten
id = sub[0]
lines = server.article(id)[3]
message = message_from_string('\n'.join(lines))
title = message['subject']
body = message.get_payload()
if message.is_multipart():
body = body[0]
yield NewsItem(title, body)
server.quit()
def getItems(self):
"""
书中原例getItems()方法
返回 nntplib.NNTPTemporaryError: 480 NEWNEWS command disabled by administrator
#480管理员禁用NEWNEWS命令
def getItems(self):
start = localtime(time() - self.window*day)
date = strftime('%y%m%d', start)
hour = strftime('%H%M%S', start)
server = NNTP(self.servername)
ids = server.newnews(self.group, date, hour)[1]
for id in ids:
lines = serverarticle(id)[3]
message = message_from_string('\n'.join(lines))
title = message['subject']
body = message.get_payload()
if message.is_multipat():
body = body[0]
yield NewsItem(title, body)
server.quit()
"""
server = NNTP(self.servername)
(resp, count, frist, last, name) = server.group(self.group)
(resp, subs) = server.xhdr('subject', (str(frist) + '-' + (last)))
for subject in subs[-10:]:
title = subject[1]
(reply, num, id, list) = server.body(subject[0])
body = ''.join(list)
#print(num) #186919
#print(title) #Re: Find out which module a class came from
#print(''.join(list))#prano wrote:> But for merely ordinary obfuscation caused by poor...
yield NewsItem(title, body)
server.quit()
class NNTPConnector(BaseConnector):
@logit(log,'fetch')
def fetch(self):
"""
Fetches all the messages for a given news group uri and return Fetched staus depending
on the success and faliure of the task
"""
try:
#eg. self.currenturi = nntp://msnews.microsoft.com/microsoft.public.exchange.setup
#nntp_server = 'msnews.microsoft.com'
#nntp_group = 'microsoft.public.exchange.setup'
self.genre = 'review'
try:
nntp_server = urlparse(self.currenturi)[1]
except:
log.exception(self.log_msg("Exception occured while connecting to NNTP server %s"%self.currenturi))
return False
nntp_group = urlparse(self.currenturi)[2][1:]
self.server = NNTP(nntp_server)
try:
self.__updateParentSessionInfo()
resp, count, first, last, name = self.server.group(nntp_group)
last_id = int(last)
first_id = self.__getMaxCrawledId(last_id)+1
log.debug("first_id is %d:"%first_id)
log.debug("last_id is %d:"%last_id)
if last_id >= first_id:
resp, items = self.server.xover(str(first_id), str(last_id))
log.debug(self.log_msg("length of items:%s"%str(len(items))))
for self.id, self.subject, self.author, self.date, self.message_id,\
self.references, size, lines in items:
self.__getMessages(self.task.instance_data['uri'])
self.server.quit()
return True
except:
log.exception(self.log_msg("Exception occured in fetch()"))
self.server.quit()
return False
except Exception,e:
log.exception(self.log_msg("Exception occured in fetch()"))
return False
def getItems(self): # 新闻生成器
yesterday = date.today() - timedelta(days=self.window) # 计算新闻获取的起始时间
server = NNTP(self.server_name) # 创建服务器连接对象
ids = server.newnews(self.group, yesterday)[1] # 获取新闻id列表
count = 0 # 创建计数变量
for id in ids: # 循环获取新闻id
count += 1 # 计数递增
if count <= 10: # 如果计数小于10
article = server.article(id)[1][2] # 获取指定id的新闻文章
lines = [] # 创建每行新闻内容的列表
for line in article: # 从新闻文章中读取每一行内容
lines.append(line.decode()) # 将每行新闻内容解码，添加到新闻内容列表。
message = message_from_string('\n'.join(lines)) # 合并新闻列表内容为字符串并转为消息对象
title = message['subject'].replace('\n', '') # 从消息对象中获取标题
body = message.get_payload() # 从消息对象中获取到新闻主体内容
if message.is_multipart(): # 如果消息对象包含多个部分
body = body[0] # 获取到的内容中第1个部分获取新闻主体内容
yield NewsItem(title, body) # 生成1个新闻内容对象
else: # 如果超出10条内容
break # 跳出循环
server.quit() # 关闭连接
class DanskGruppenArchive(object):
"""Class that provides an interface to Dansk-gruppens emails archive on
gmane
"""
def __init__(self, article_cache_size=300, cache_file=None):
"""Initialize local variables"""
# Connect to news.gmane.org
self.nntp = NNTP('news.gmane.org')
# Setting the group returns information, which right now we ignore
self.nntp.group('gmane.comp.internationalization.dansk')
# Keep a local cache in an OrderedDict, transferred across session
# in a pickled version in a file
self.article_cache_size = article_cache_size
self.cache_file = cache_file
if cache_file and path.isfile(cache_file):
with open(cache_file, 'rb') as file_:
self.article_cache = pickle.load(file_)
logging.info('Loaded %i items from file cache',
len(self.article_cache))
else:
self.article_cache = OrderedDict()
def close(self):
"""Quit the NNTP session and save the cache"""
self.nntp.quit()
if self.cache_file:
with open(self.cache_file, 'wb') as file_:
pickle.dump(self.article_cache, file_)
logging.info('Wrote %i items to cache file',
len(self.article_cache))
@property
def last(self):
"""Return the last NNTP ID as an int"""
return self.nntp.group('gmane.comp.internationalization.dansk')[3]
def _get_article(self, message_id):
"""Get an article (cached)
Args:
message_id (int): The NNTP ID of the message
Returns:
list: List of byte strings in the message
"""
# Clear excess cache
if len(self.article_cache) > self.article_cache_size:
self.article_cache.popitem(last=False)
# Check if article is in cache and if not, put it there
if message_id not in self.article_cache:
# nntp.article() returns: response, information
# pylint: disable=unbalanced-tuple-unpacking
_, info = self.nntp.article(message_id)
self.article_cache[message_id] = info
return self.article_cache[message_id]
@staticmethod
def _article_to_email(article):
"""Convert a raw article to an email object
Args:
article (namedtuple): An article named tuple as returned by NNTP
Returns:
email.message: An email message object
"""
# article lines are a list of byte strings
decoded_lines = [line.decode('ascii') for line in article.lines]
article_string = '\n'.join(decoded_lines)
# Make an email object
return email.message_from_string(article_string)
def get_subject(self, message_id):
"""Get the subject of an message
Args:
message_id (int): The NNTP ID of the the message
Returns:
str: The subject of the article
"""
article = self._get_article(message_id)
mail = self._article_to_email(article)
# The subject may be encoded by NNTP, so decode it
return decode_header(mail['Subject'])
def get_body(self, message_id):
"""Get the body of a message
Args:
message_id (int): The NNTP ID of the the message
Returns:
str: The body of the article as a str or None if no body could be
found or succesfully decoded
"""
article = self._get_article(message_id)
mail = self._article_to_email(article)
# Walk parts of the email and look for text/plain content type
for part in mail.walk():
if part.get_content_type() == 'text/plain':
body = part.get_payload(decode=True)
# Find the text encoding from lines like:
# text/plain; charset=UTF-8
# text/plain; charset=utf-8; format=flowed
# Encoding sometimes has "" around it, decode is OK with that
for type_part in part['Content-Type'].split(';'):
if type_part.strip().startswith('charset='):
encoding = type_part.replace('charset=', '')
break
else:
message = 'Looking for the character encoding in the '\
'string "%s" went wrong'
logging.warning(message, part['Content-Type'])
return None
# Decode and return the body
try:
body = body.decode(encoding)
except LookupError:
message = 'Do not know how to handle a body with '\
'charset: %s'
logging.warning(message, encoding)
return None
return body
def get_attachment(self, message_id, filename):
"""Get attachment by filename
Args:
message_id (int): The NNTP ID of the the message
filename (str): The filename for the attachment
Returns:
bytes: The binary content of the attachment
"""
return self.get_attachments(message_id).get(filename)
def get_attachments(self, message_id):
"""Get attachments
Args:
message_id (int): The NNTP ID of the the message
Returns:
dict: Dict with attachments where keys are filenames and values are
their binary content
"""
article = self._get_article(message_id)
mail = self._article_to_email(article)
attachments = {}
# Walk parts of the email and look for application/octet-stream
# content type
for part in mail.walk():
content_disp = part['Content-Disposition']
if not (content_disp and content_disp.startswith('attachment')):
continue
# Get the filename from a line like: Content-Disposition:
# attachment; filename="hitori.master.da.podiff"
filename = None
for disp_part in content_disp.split(';'):
if disp_part.strip().startswith('filename='):
filename = disp_part.strip().replace('filename=', '')
# Strip " from filename
filename = filename.strip('"')
if filename is None:
message = 'Unable to extract filename from '\
'Content-Disposition: %s'
logging.warning(message, part['Content-Disposition'])
raise Exception('Unable to extract filename')
attachments[filename] = part.get_payload(decode=True)
return attachments
#! /usr/bin/python from sys import stdin from nntplib import NNTP from os import environ s = NNTP(environ["NNTPSERVER"]) s.post(stdin) s.quit()
oneDay = 24 * 60 * 60
yesterday = datetime.now() - timedelta(days=1)
#date = strftime('%y%m%d', yesterday)
#hour = strftime('%H%M%S', yesterday)
servername = 'news.gmane.org'
groupname = 'gmane.comp.python.committers'
server = NNTP(servername)
resp, count, first, last, name = server.group(groupname)
resp, overviews = server.over((last -9, last))
for id, over in overviews:
print(id, nntplib.decode_header(over['subject']))
'''
ids = server.newnews(group, yesterday)[1]
for id in ids:
head = server.head(id)[3]
for line in head:
if line.lower().startswith('subject:'):
subject = line[9:]
break
body = server.body(id)[3]
print(subject)
print('-'*len(subject))
print('\n'.join(body))
server.quit()
#__author: ZhengNengjin
#__date: 2018/10/25
from nntplib import NNTP
n = NNTP('your.nntp.server')
rsp, ct, fst, lst, grp = n.group('comp,lang.python')
rsp, anum, mid, data = n.article('110457')
for eachLine in data:
print(eachLine)
n.quit()
def readnews(I="", A=None, P=None, RESPONSE=None):
"""Display article in HTML"""
article = I
user = A
password = P
RESPONSE.write("""<HTML><HEAD><TITLE>Tokyo PC Users Group</TITLE></HEAD>
<BODY BGCOLOR="#FFFFFF">""")
try:
news = NNTP(NEWS_SERVER)
except:
RESPONSE.write("Can not connect to server: " + NEWS_SERVER)
resp = news.shortcmd('MODE READER')
if user:
resp = news.shortcmd('authinfo user '+user)
if resp[:3] == '381':
if not password:
RESPONSE.write("<B>Can not fetch article</B><P>")
else:
resp = news.shortcmd('authinfo pass '+password)
if resp[:3] != '281':
RESPONSE.write("<B>Can not fetch article</B><P>")
try:
resp, nr, id, subs = news.head(article)
except:
RESPONSE.write("Article %s not available" % quote(article))
RESPONSE.write('<TABLE WIDTH="100%" BGCOLOR="#CFCFCF"><TR><TD>')
# build up the header (so we know Subject: by Newsgroups: output time)
from_line = ""
newsgroup_line = ""
subject_line = ""
keep_lines = ""
mail_subject = ""
for line in subs:
ihdr = interesting_headers.match(line)
if ihdr:
if ihdr.group('from'):
name, email = parseaddr(line[6:])
if name:
from_line = 'From: <A HREF="mailto:%s%s">%s</A> <%s><BR>' % (
email, "%s", name, email)
else:
from_line = 'From: <A HREF="mailto:%s%s">%s</A><BR>' % (
email, "%s", email)
elif ihdr.group('newsgroups'):
newsgroup_line = 'Newsgroups: <A HREF="mailto:tpc-%[email protected]%s">tpc.%s</A>%s<BR>' % (
ihdr.group('group'), "%s",
ihdr.group('group'), ihdr.group('othergroups'))
elif ihdr.group('subject'):
subject_line = 'Subject: <B>%s</B><BR>' % line[9:]
if ihdr.group('re'):
mail_subject = "?subject="+line[9:]
else:
mail_subject = "?subject=Re: "+line[9:]
elif ihdr.group('keep'):
keep_lines = keep_lines+line+"<BR>"
if from_line:
RESPONSE.write(from_line % mail_subject)
if newsgroup_line:
RESPONSE.write(newsgroup_line % mail_subject)
RESPONSE.write(subject_line + keep_lines)
RESPONSE.write('</TD></TR></TABLE><P>')
try:
resp, nr, id, subs = news.body(article)
except:
RESPONSE.write("Article %s body not available" % article)
RESPONSE.write("<CODE>")
for line in subs:
RESPONSE.write(re.sub(r'''(?i)(?x)
(?P<opening>[<(";]?)
(?P<url>(((?P<web>http:)|(news:)|(mailto:)|(telnet:))(?P<uri>\S*?))
# a mail address is some non-ws characters followed by @
# followed by a domain name that has at least one . in it
|(?P<mailadr>\S+@(\S+\.)+\S+?))
# either a URL or a mail address will not contain [)">\s]
# and will not end with punctuation just before the whitespace
(?P<closing>([)"'>\s]|$|([&.?,:;]\s)+))''',
liven_url, line) + "<BR>")
RESPONSE.write("</CODE>")
RESPONSE.write("</BODY></HTML>")
resp = news.quit()
class DownloadSpots(object):
def __init__(self):
try:
self.news = NNTP(NEWS_SERVER, NNTP_PORT, NNTP_USERNAME,
NNTP_PASSWORD)
except NNTPTemporaryError as e:
raise SpotError('NNTP', e)
except socket.error as e:
raise SpotError('Connection', e)
self.conn = sqlite3.connect(NEWS_SERVER + '.db')
self.conn.row_factory = sqlite3.Row
self.cur = self.conn.cursor()
self.cur.executescript('''\
PRAGMA synchronous = OFF;
PRAGMA journal_mode = MEMORY;
PRAGMA temp_store = MEMORY;
PRAGMA count_changes = OFF;
''')
def __del__(self):
print 'quit!'
if hasattr(self, 'news'):
self.news.quit()
if hasattr(self, 'cur'):
self.cur.close()
if hasattr(self, 'conn'):
self.conn.close()
def make_table(self):
sql = '''\
CREATE TABLE IF NOT EXISTS spots (
id int PRIMARY KEY,
full_id str,
cat int,
title str,
poster str,
date int,
size int,
erotiek int,
subcats str,
modulus int,
keyid int,
c_count int
);
CREATE TABLE IF NOT EXISTS comments (
id int PRIMARY KEY,
full_id str,
spot_full_id str
);
CREATE INDEX IF NOT EXISTS spots_full_id_index on spots(full_id);
'''
self.cur.executescript(sql)
self.conn.commit()
def download_detail(self, id):
'''Get information about a spot.
Args:
id (int/string): the nntp id of the spot. Can be:
- a short id: 123456
- a long id: '*****@*****.**'
Returns:
a dict with the following keys:
- nzb: a list of nntp id's of the nzb file
- image: the location of the image file: url or nntp id
- website: url of the website
- desc: a description of the spot
- title2: the title of the spot
(is called title2 to avoid a conflict)
'''
id = str(id)
self.news.group('free.pt')
print id
head = self.news.head(id)[-1]
xmltext = ''.join(item[7:] for item in head if
item.startswith('X-XML:'))
xml = etree.XML(xmltext)
xmldict = defaultdict(list)
xmldict['nzb'] = [i.text for i in xml.find('.//NZB').iter('Segment')]
imgfind = xml.find('.//Image')
if imgfind is not None:
if imgfind.find('.//Segment') is not None:
xmldict['image'] = imgfind.find('.//Segment').text
else:
xmldict['image'] = imgfind.text
else:
xmldict['image'] = None
webfind = xml.find('.//Website')
if webfind is not None:
xmldict['website'] = webfind.text
else:
xmldict['website'] = None
xmldict['desc'] = bb2html('\n'.join(self.news.body(id)[-1]))
xmldict['title2'] = xml.find('.//Title').text
print xmldict
return xmldict
def download_image(self, article, imgnr):
'''Download and save an image file.
Args:
article: Location of the file, can be:
- None
- an url: 'http://url.to.image.ext'
- a nntp id: '*****@*****.**'
imgnr: filename of the saved image
'''
print imgnr
file = 'temp/%s.picture' % imgnr
if article is None:
shutil.copy('none.png', file)
elif article.startswith('http'):
if 'imgchili.com' in article:
article = re.sub(r't([0-9]\.imgchili)', r'i\1', article)
try:
print urllib.urlretrieve(article, file)
except:
print 'Image download error'
shutil.copy('none.png', file)
elif '@' in article:
article = '<%s>' % article
print article
try:
self.news.group('alt.binaries.ftd')
data = ''.join(self.news.body(article)[-1])
except NNTPTemporaryError as e:
shutil.copy('none.png', file)
raise SpotError('NNTP', e)
else:
data = data.replace('=A', '\x00').replace('=B', '\r') \
.replace('=C', '\n').replace('=D', '=')
with open(file, 'wb') as f:
f.write(data)
else:
shutil.copy('none.png', file)
def download_nzb(self, articles, title):
'''Download and save a nzb file.
Args:
articles: a list of nntp id's: ['*****@*****.**']
title (string): the filename of the saved nzb file
(must be already escaped)
'''
print 'download_nzb'
file = 'temp/%s.nzb' % title
self.news.group('alt.binaries.ftd')
print articles
data = ''.join(''.join(self.news.body('<%s>' % article)[-1]) for
article in articles)
data = data.replace('=A', '\x00').replace('=B', '\r') \
.replace('=C', '\n').replace('=D', '=')
data = zlib.decompress(data, -zlib.MAX_WBITS)
with open(file, 'wb') as f:
f.write(data)
def update_spots(self):
'''Download new spots and save them to the database.
Yields:
total: total amount of spots to be downloaded
subtotal: amount of spots already done
'''
print 'Opening database...',
self.make_table()
self.cur.execute('SELECT count(id) FROM spots')
oude_lengte = self.cur.fetchone()[0]
print 'Done'
print oude_lengte, 'spots in database'
print 'Connecting...',
last = int(self.news.group('free.pt')[3])
print 'Done'
if oude_lengte:
self.cur.execute('SELECT max(id) FROM spots')
current = self.cur.fetchone()[0]
else:
current = last - (NR_OF_SPOTS * 100)
delta = 2000
total = last - current
print 'Current:', current
print 'Last:', last
print total, 'spots op te halen'
yield total
if current >= last:
print 'Geen nieuwe spots!'
yield 'end'
return
print 'Getting new spots...',
self.cur.execute('DROP INDEX IF EXISTS spots_full_id_index')
self.conn.commit()
for i in xrange(0, total, delta):
for (id, subject), (id, fromline), (id, full_id) in \
zip(*(self.news.xhdr(header, '%s-%s'
% (current + 1 + i, current + delta + i))[-1] for
header in ('subject', 'from', 'Message-ID'))):
subject = re.sub('^[\t]', '', subject).split('\t')[0]
if subject[:7] == 'DISPOSE':
continue
subject = subject.decode('latin-1')
fromline = fromline.decode('latin-1')
subject = re.sub('\s\|\s\w+$', '', subject)
poster = re.sub('(\t[^\t]+\t)?([^\s]+)\s<.+$', r'\2', fromline)
try:
info = re.findall(r'<[^>]+>', fromline)[0].split('@')
modulus = info[0]
info = info[1].split('.')[:4]
date = info[3]
except IndexError:
continue
if info[0][1] == '7':
modulus = modulus.split('.')[0][1:]
else:
modulus = 0
subcats = '|'.join(info[0][2:][x:x + 3] for x in
xrange(0, len(info[0][2:]), 3))
erotiek = 0
if info[0][0] == '1':
for genre in re.findall('d[0-9]{2}', subcats):
if ((23 <= int(genre[1:]) <= 26) or
(72 <= int(genre[1:]) <= 89)):
erotiek = 1
break
sql = ('INSERT INTO spots (id, full_id, cat, title, poster, date, '
'size, erotiek, subcats, modulus, keyid, c_count) '
'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)')
self.cur.execute(sql, (id, full_id, info[0][0], subject, poster,
date, info[1], erotiek, subcats, modulus,
info[0][1]))
self.conn.commit()
yield int(id) - current # subtotal
self.cur.execute('CREATE INDEX spots_full_id_index on spots(full_id)')
self.conn.commit()
print 'Spots klaar!'
self.cur.execute('SELECT count(id) FROM spots')
lengte = self.cur.fetchone()[0]
print lengte - oude_lengte, 'nieuwe spots!'
self.news.quit()
yield 'end'
def update_comments(self):
'''Download new comments and save them to the database.
Yields:
total: total amount of comments to be downloaded
subtotal: amount of comments already done
'''
print 'comments'
print 'Opening database...',
self.make_table()
self.cur.execute('SELECT count(id) FROM comments')
oude_lengte = self.cur.fetchone()[0]
print 'Done'
print oude_lengte, 'comments in database'
print 'Connecting...',
last = int(self.news.group('free.usenet')[3])
print 'Done'
if oude_lengte:
self.cur.execute('SELECT max(id) FROM comments')
current = self.cur.fetchone()[0]
print current
else:
current = last - (NR_OF_SPOTS * 500)
delta = 2000
total = last - current
print 'Current:', current
print 'Last:', last
print total, 'comments op te halen'
yield total
if current >= last:
print 'Geen nieuwe comments!'
yield 'end'
return
print 'begin reacties', time.time()
for i in xrange(0, total, delta):
for (id, ref), (id, msgid) in \
zip(*(self.news.xhdr(header, '%s-%s'
% (current + 1 + i, current + delta + i))[-1] for
header in ('References', 'Message-ID'))):
self.cur.execute('INSERT INTO comments VALUES (?, ?, ?)',
(id, msgid, ref))
self.cur.execute('UPDATE spots SET c_count = c_count + 1 WHERE full_id = ?',
(ref,))
yield int(id) - current # subtotal
self.conn.commit()
print 'Reacties klaar'
self.cur.execute('SELECT count(id) FROM comments')
lengte = self.cur.fetchone()[0]
print lengte - oude_lengte, 'nieuwe comments!'
self.news.quit()
yield 'end'
def show(self, search_col, op, search, search_rest='', limit=NR_OF_SPOTS,
columns='*'):
'''Perform a sql query.
Args:
search_col: the column to use in the where clause
op: the operator to use in the where clause
search: the search value to use in the where clause
search_rest: additional conditions for the where clause
limit: the number of rows to select
columns: the columns to select
Returns:
a list of tuples with the results of the sql query
'''
self.make_table()
sql = '''\
SELECT %s FROM spots
WHERE %s %s :search %s
ORDER BY id DESC
LIMIT %s
''' % (columns, search_col, op, search_rest, limit)
results = map(Spot, self.cur.execute(sql, (search,)))
print len(results)
return results
import nntplib
from nntplib import NNTP
s = NNTP('news.gmane.org')
resp, count, first, last, name = s.group('gmane.comp.python.committers')
print ('Group', name, 'has', count, 'articles, range', first, 'to', last)
resp, subs = s.xhdr('subject',first)
for id, sub in subs[-10:]: print (id, sub)
try :
for id, sub in subs[-10:]: print (id, sub)
except TypeError as e:
print(e)
print(s.quit())
# NNTP.
class DanskGruppenArchive(object):
"""Class that provides an interface to Dansk-gruppens emails archive on
gmane
"""
def __init__(self, article_cache_size=300, cache_file=None):
"""Initialize local variables"""
# Connect to news.gmane.org
self.nntp = NNTP('news.gmane.org')
# Setting the group returns information, which right now we ignore
self.nntp.group('gmane.comp.internationalization.dansk')
# Keep a local cache in an OrderedDict, transferred across session
# in a pickled version in a file
self.article_cache_size = article_cache_size
self.cache_file = cache_file
if cache_file and path.isfile(cache_file):
with open(cache_file, 'rb') as file_:
self.article_cache = pickle.load(file_)
logging.info('Loaded %i items from file cache',
len(self.article_cache))
else:
self.article_cache = OrderedDict()
def close(self):
"""Quit the NNTP session and save the cache"""
self.nntp.quit()
if self.cache_file:
with open(self.cache_file, 'wb') as file_:
pickle.dump(self.article_cache, file_)
logging.info('Wrote %i items to cache file',
len(self.article_cache))
@property
def last(self):
"""Return the last NNTP ID as an int"""
return self.nntp.group('gmane.comp.internationalization.dansk')[3]
def _get_article(self, message_id):
"""Get an article (cached)
Args:
message_id (int): The NNTP ID of the message
Returns:
list: List of byte strings in the message
"""
# Clear excess cache
if len(self.article_cache) > self.article_cache_size:
self.article_cache.popitem(last=False)
# Check if article is in cache and if not, put it there
if message_id not in self.article_cache:
# nntp.article() returns: response, information
# pylint: disable=unbalanced-tuple-unpacking
_, info = self.nntp.article(message_id)
self.article_cache[message_id] = info
return self.article_cache[message_id]
@staticmethod
def _article_to_email(article):
"""Convert a raw article to an email object
Args:
article (namedtuple): An article named tuple as returned by NNTP
Returns:
email.message: An email message object
"""
# article lines are a list of byte strings
decoded_lines = [line.decode('ascii') for line in article.lines]
article_string = '\n'.join(decoded_lines)
# Make an email object
return email.message_from_string(article_string)
def get_subject(self, message_id):
"""Get the subject of an message
Args:
message_id (int): The NNTP ID of the the message
Returns:
str: The subject of the article
"""
article = self._get_article(message_id)
mail = self._article_to_email(article)
# The subject may be encoded by NNTP, so decode it
return decode_header(mail['Subject'])
def get_body(self, message_id):
"""Get the body of a message
Args:
message_id (int): The NNTP ID of the the message
Returns:
str: The body of the article as a str or None if no body could be
found or succesfully decoded
"""
article = self._get_article(message_id)
mail = self._article_to_email(article)
# Walk parts of the email and look for text/plain content type
for part in mail.walk():
if part.get_content_type() == 'text/plain':
body = part.get_payload(decode=True)
# Find the text encoding from lines like:
# text/plain; charset=UTF-8
# text/plain; charset=utf-8; format=flowed
# Encoding sometimes has "" around it, decode is OK with that
for type_part in part['Content-Type'].split(';'):
if type_part.strip().startswith('charset='):
encoding = type_part.replace('charset=', '')
break
else:
message = 'Looking for the character encoding in the '\
'string "%s" went wrong'
logging.warning(message, part['Content-Type'])
return None
# Decode and return the body
try:
body = body.decode(encoding)
except LookupError:
message = 'Do not know how to handle a body with '\
'charset: %s'
logging.warning(message, encoding)
return None
return body
def get_attachment(self, message_id, filename):
"""Get attachment by filename
Args:
message_id (int): The NNTP ID of the the message
filename (str): The filename for the attachment
Returns:
bytes: The binary content of the attachment
"""
return self.get_attachments(message_id).get(filename)
def get_attachments(self, message_id):
"""Get attachments
Args:
message_id (int): The NNTP ID of the the message
Returns:
dict: Dict with attachments where keys are filenames and values are
their binary content
"""
article = self._get_article(message_id)
mail = self._article_to_email(article)
attachments = {}
# Walk parts of the email and look for application/octet-stream
# content type
for part in mail.walk():
content_disp = part['Content-Disposition']
if not (content_disp and content_disp.startswith('attachment')):
continue
# Get the filename from a line like: Content-Disposition:
# attachment; filename="hitori.master.da.podiff"
filename = None
for disp_part in content_disp.split(';'):
if disp_part.strip().startswith('filename='):
filename = disp_part.strip().replace('filename=', '')
# Strip " from filename
filename = filename.strip('"')
if filename is None:
message = 'Unable to extract filename from '\
'Content-Disposition: %s'
logging.warning(message, part['Content-Disposition'])
raise Exception('Unable to extract filename')
attachments[filename] = part.get_payload(decode=True)
return attachments
def identify(host):
h = NNTP(host)
w = h.getwelcome()
h.quit()
return patterns.match(w)
#!/usr/bin/python
from nntplib import NNTP
from time import time, localtime, strftime
day = 24 * 60 * 60
yesterday = localtime(time() - day)
date = strftime('%y%m%d', yesterday)
t = strftime('%H%M%S', yesterday)
s = NNTP('web.aioe.org')
g = 'comp.lang.python.announce'
ids = s.newnews(g, date, t)[1]
for id in ids:
head = s.head(id)[3]
for line in head:
if line.lower().startswith('subject:'):
subject = line[9:]
break
body = s.body(id)[3]
print subject
print '-' * len(subject)
print '\n'.join(body)
s.quit()
def get_id(): #创建新闻id生成器
ids = server.newnews(group, yesterday)[1] #获取进近4小时新闻内容中的所有新闻id
for id in ids: #遍历所有新闻id
yield id #生成1个新闻id
ids = get_id() #创建新闻id生成器对象
id = next(ids) #获取第一个新闻id
head_data = server.head(id)[1][2] #获取新闻头部内容
body_data = server.body(id)[1][2] #获取新闻的主体内容
title = '' #创建标题
body = '' #创建主体
for line in head_data: #遍历头部内容
if line.decode().lower().startswith(
'subject:'): #如果发现新闻标题特征（"subject:"开头） startswithk开始于
title = line[9:].decode() #去除特征文字保存标题内容
for line in body_data: #遍历主体内容
if line.decode().endswith('='): #如果行内容以'='结尾 endswith结束于
line = line[:-1] #去除“=”
if line.decode().endswith('=20'): #如果行内容以'=20'结尾
line = line[:-3] + b'\n' #去除'=20'并添加换行符
body += line.decode() #将每行内容组织为新的主体内容
print(title) #显示输出标题内容
print('-' * len(title)) #显示输出和标题字符数量同等的'-'符号
print(body) #显示输出主体内容
server.quit() #退出与服务器的连接
servername = 'news.gmane.org'
groupname = 'gmane.comp.python.general' # cmd line args or defaults
showcount = 10 # show last showcount posts
# connect to nntp server
print 'Connecting to', servername, 'for', groupname
from nntplib import NNTP
connection = NNTP(servername)
(reply, count, first, last, name) = connection.group(groupname)
print '%s has %s articles: %s-%s' % (name, count, first, last)
# get request headers only
fetchfrom = str(int(last) - (showcount-1))
(reply, subjects) = connection.xhdr('subject', (fetchfrom + '-' + last))
# show headers, get message hdr+body
for (id, subj) in subjects: # [-showcount:] if fetch all hdrs
print 'Article %s [%s]' % (id, subj)
if not listonly and raw_input('=> Display?') in ['y', 'Y']:
reply, num, tid, list = connection.head(id)
for line in list:
for prefix in showhdrs:
if line[:len(prefix)] == prefix:
print line[:80]; break
if raw_input('=> Show body?') in ['y', 'Y']:
reply, num, tid, list = connection.body(id)
for line in list:
print line[:80]
print
print connection.quit( )
groupname = 'comp.lang.python' # cmd line args or defaults
showcount = 10 # show last showcount posts
# connect to nntp server
print 'Connecting to', servername, 'for', groupname
from nntplib import NNTP
connection = NNTP(servername)
(reply, count, first, last, name) = connection.group(groupname)
print '%s has %s articles: %s-%s' % (name, count, first, last)
# get request headers only
fetchfrom = str(int(last) - (showcount-1))
(reply, subjects) = connection.xhdr('subject', (fetchfrom + '-' + last))
# show headers, get message hdr+body
for (id, subj) in subjects: # [-showcount:] if fetch all hdrs
print 'Article %s [%s]' % (id, subj)
if not listonly and raw_input('=> Display?') in ['y', 'Y']:
reply, num, tid, list = connection.head(id)
for line in list:
for prefix in showhdrs:
if line[:len(prefix)] == prefix:
print line[:80]; break
if raw_input('=> Show body?') in ['y', 'Y']:
reply, num, tid, list = connection.body(id)
for line in list:
print line[:80]
print
print connection.quit()
groupname = 'comp.lang.python' # cmd line args or defaults #组名
showcount = 10 # show last showcount posts #显示文章的个数
# connect to nntp server
print('Connecting to', servername, 'for', groupname)
from nntplib import NNTP
connection = NNTP(servername)
(reply, count, first, last, name) = connection.group(groupname)
print('%s has %s articles: %s-%s' % (name, count, first, last))
# get request headers only
fetchfrom = str(int(last) - (showcount-1))
(reply, subjects) = connection.xhdr('subject', (fetchfrom + '-' + last))
# show headers, get message hdr+body
for (id, subj) in subjects: # [-showcount:] if fetch all hdrs
print('Article %s [%s]' % (id, subj))
if not listonly and input('=> Display?') in ['y', 'Y']: #询问是否打印文章
reply, num, tid, list = connection.head(id)
for line in list:
for prefix in showhdrs:
if line[:len(prefix)] == prefix:
print(line[:80])
break
if input('=> Show body?') in ['y', 'Y']:
reply, num, tid, list = connection.body(id)
for line in list:
print(line[:80])
print()
print(connection.quit())
# -*- coding:utf-8 -*-
from nntplib import NNTP
servername = 'web.aioe.org'
group = 'comp.lang.python.announce'
server = NNTP(servername)
howmany = 10
resp, count, first, last, name = server.group(group)
start = last - howmany + 1
resp, overviews = server.over((start, last))
for id, over in overviews:
subject = over['subject']
resp, info = server.body(id)
print('subject:', subject)
print('-' * len(subject))
for line in info.lines:
print(line.decode('latin1'))
print()
server.quit()
# соединиться с сервером NNTP
print('Connecting to', servername, 'for', groupname)
from nntplib import NNTP
connection = NNTP(servername)
(reply, count, first, last, name) = connection.group(groupname)
print('%s has %s articles: %s-%s' % (name, count, first, last))
# запросить только заголовки
fetchfrom = str(int(last) - (showcount - 1))
(reply, subjects) = connection.xhdr('subject', (fetchfrom + '-' + last))
# вывести заголовки, получить заголовки + тело
for (id, subj) in subjects: # [-showcount:] для загрузки всех заголовков
print('Article %s [%s]' % (id, subj))
if not listonly and input('=> Display?') in ['Y', 'y']:
reply, num, tid, listing = connection.head(id)
for line in listing:
for prefix in showhdrs:
if line[:len(prefix)] == prefix:
print(line[:80])
break
if input('=> Show body?') in ['Y', 'y']:
reply, num, tid, listing = connection.body(id)
for line in listing:
print(line[:80])
print()
print(connection.quit())
class NewsWatcher(MessageWatcher):
def __init__(self, server, groups,
user=None, pw=None, port=None, tag=None):
MessageWatcher.__init__(self)
self.server = server
self.groups = groups
self.nntp = None # the NNTP connection object
self.user = user
self.pw = pw
self.port = port
self.tag = tag
self.last = {}
self.timeout = None
self.pollInterval = 60
self.debug = 0
def __repr__(self):
return "<NewsWatcher %s:%s (%s)>" % (self.server, self.port,
",".join(self.groups))
def __getstate__(self):
d = MessageWatcher.__getstate__(self)
d['nntp'] = None # just in case
return d
def start(self):
port = self.port
if not port:
port = NNTP_PORT
self.nntp = NNTP(self.server, port, self.user, self.pw,
readermode=1)
# only look for messages that appear after we start. Usenet is big.
if not self.last: # only do this the first time
for g in self.groups:
resp, count, first, last, name = self.nntp.group(g)
self.last[g] = int(last)
if self.debug: print "last[%s]: %d" % (g, self.last[g])
self.timeout = gtk.timeout_add(self.pollInterval*1000,
self.doTimeout)
def stop(self):
self.nntp.quit()
self.nntp = None
if self.timeout:
gtk.timeout_remove(self.timeout)
self.timeout = None
def doTimeout(self):
self.poll()
return gtk.TRUE # keep going
def poll(self):
#print "polling", self
for g in self.groups:
resp, count, first, last, name = self.nntp.group(g)
for num in range(self.last[g]+1, int(last)+1):
try:
resp, num, id, lines = self.nntp.article("%d" % num)
except NNTPError:
continue
name = "%s:%d" % (g, int(num))
if self.debug: print "got", name
if self.tag:
if not filter(lambda line, tag=tag: line.find(tag) != -1,
lines):
continue
self.parseMessage(name, name, time.time(), lines)
self.last[g] = int(last)
print eachLine From: "Alex Martelli" <alex@...> Subject: Re: Rounding Question Date: Wed, 21 Feb 2001 17:05:36 +0100 "Remco Gerlich" <remco@...> wrote: Jacob Kaplan-Moss <jacob@...> wrote in comp.lang.python: So I've got a number between 40 and 130 that I want to round up to the nearest 10. That is: 40 --> 40, 41 --> 50, ..., 49 --> 50, 50 --> 50, 51 --> 60 Rounding like this is the same as adding 5 to the number and then rounding down. Rounding down is substracting the remainder if you were to divide by 10, for which we use the % operator in Python. This will work if you use +9 in each case rather than +5 (note that he doesn't really want rounding -- he wants 41 to 'round' to 50, for ex). Alex >>> n.quit() '205 closing connection - goodbye!'
