def __init__(self, host, port, group, user, password):
logging.info('Initializing NNTP client')
if port:
self.nntp = nntplib.NNTP(host, port, user=user, password=password)
else:
self.nntp = nntplib.NNTP(host, user=user, password=password)
self.group = group
def connect(self):
try:
if self.user and self.password:
self._connection = nntplib.NNTP(self.url,
user=self.user,
password=self.password,
timeout=self.timeout)
else:
self._connection = nntplib.NNTP(self.url, timeout=self.timeout)
except:
self._dead = True
def newssearch(self, sname):
'''Retrieves messages from newsserver.'''
self.ui.note('searching news server %s\n' % sname)
try:
nserv = nntplib.NNTP(sname, readermode=True, usenetrc=True)
except (nntplib.NNTPPermanentError, nntplib.NNTPTemporaryError):
try:
nserv = nntplib.NNTP(sname, readermode=True, usenetrc=False)
except (nntplib.NNTPPermanentError,
nntplib.NNTPTemporaryError), inst:
self.ui.warn('%s\n' % inst)
return
def set_nntp(self):
n = nntplib.NNTP(self.host, self.port, "", "", True)
resp, count, first, last, name = n.group(self.groupname)
self._nntp = n
self._count = int(count)
self._first = int(first)
self._last = int(last)
def _connect(self):
"""Connects to a remote Usenet server"""
LOGGER.info('New remote connection')
connection = nntplib.NNTP(self.configs['host'],
user=self.configs['user'],
password=self.configs['password'])
return connection
def fetch(segment, groups) -> None:
host = flags['h']
port = None
if ':' in host:
host, port = tuple(host.split(':', 1))
port = int(port)
try:
if flags['ssl']:
if not port:
port = 563
s = nntplib.NNTP_SSL(host, port)
else:
if not port:
port = 119
s = nntplib.NNTP(host, port)
s.login(flags['u'], flags['p'])
for group in groups:
try:
s.group(group)
watchdog.put(segment)
s.body('<{}>'.format(segment), file=segment)
except nntplib.NNTPTemporaryError:
continue
break
s.quit()
except nntplib.NNTPPermanentError as e:
log('slurp: nntp: ' + str(e))
def main(): try: n=nntplib.NNTP(HOST) #user=USER,pwd=PASS except socket.gaierror,e: print 'error,can not reach host',HOST return
def _connect_nntp(self, nntpserver):
"""
This is done only once per checking task. Also, the newly
introduced error codes 504 and 505 (both inclining "Too busy, retry
later", are caught.
"""
tries = 0
nntp = None
while tries < 2:
tries += 1
try:
nntp = nntplib.NNTP(nntpserver, usenetrc=False)
except nntplib.NNTPTemporaryError:
self.wait()
except nntplib.NNTPPermanentError as msg:
if re.compile("^50[45]").search(str(msg)):
self.wait()
else:
raise
if nntp is None:
raise LinkCheckerError(
_("NNTP server too busy; tried more than %d times.") % tries)
if log.is_debug(LOG_CHECK):
nntp.set_debuglevel(1)
self.add_info(nntp.getwelcome())
return nntp
def main() :
try:
n = nntplib.NNTP(HOST)
#
try:
resp, cnt, first, last, grp = n.group(GRNM)
except nntplib.NNTPPermanentError as e:
print "cannot load group %s" % GRNM
return
rng = "%s-%s" % (first, last)
resp, frms = n.xhdr("from", rng)
resp, subs = n.xhdr("subject", rng)
resp, dats = n.xhdr("date", rng)
print """*** Found last article #(%s)
Form:%s
Subject:%s
Data:%s
""" % (last, frms[0][1], subs[0][1], dats[0][1])
resp, num, id, list = n.body(last)
displayFirst20(list)
except socket.gaierror as e:
print "Error cannot reach %s" % str(e)
except nntplib.NNTPPermanentError as e:
print "Error Access denied on %s" % HOST
finally:
if n in locals():
n.quit()
def main():
try:
n = nntplib.NNTP(HOST)
except socket.gaierror, e:
print 'Error: cannot reach host "%s"' % HOST
print ' ("%s")' % eval(str(e))[1]
return
def _dispose(self, mlist, msg, msgdata):
# Make sure we have the most up-to-date state
mlist.Load()
if not msgdata.get('prepped'):
prepare_message(mlist, msg, msgdata)
try:
# Flatten the message object, sticking it in a StringIO object
fp = StringIO(msg.as_string())
conn = None
try:
try:
nntp_host, nntp_port = Utils.nntpsplit(mlist.nntp_host)
conn = nntplib.NNTP(nntp_host,
nntp_port,
readermode=True,
user=mm_cfg.NNTP_USERNAME,
password=mm_cfg.NNTP_PASSWORD)
conn.post(fp)
except nntplib.error_temp as e:
syslog('error',
'(NNTPDirect) NNTP error for list "%s": %s',
mlist.internal_name(), e)
except socket.error as e:
syslog('error',
'(NNTPDirect) socket error for list "%s": %s',
mlist.internal_name(), e)
finally:
if conn:
conn.quit()
except Exception as e:
# Some other exception occurred, which we definitely did not
# expect, so set this message up for requeuing.
self._log(e)
return True
return False
def main():
parser = get_parser()
args = parser.parse_args()
server = nntplib.NNTP('news.gmane.org')
# FIXME: extract to separate file
if args.test:
if args.article:
parser.error("--article should not be provided in test mode")
if args.url:
parser.error("a URL should not be provided in test mode")
for test_url in TESTCASES:
print("testing %s" % test_url)
rescued = rescue(parser, args, server, test_url)
print(rescued)
try:
with urllib.request.urlopen(rescued) as response:
html = response.read()
if "not found".encode("utf-8") in html:
print("!! Message ID was not found")
except urllib.error.HTTPError as he:
if he.code == 300:
print(" 300 %s" % he.reason)
return 0
elif not args.url:
parser.error("a URL is required unless in test mode")
print(rescue(parser, args, server, args.url))
return 0
def _dispose(self, mlist, msg, msgdata):
# Make sure we have the most up-to-date state
mlist.Load()
if not msgdata.get('prepped'):
prepare_message(mlist, msg, msgdata)
try:
# Flatten the message object, sticking it in a StringIO object
fp = StringIO(msg.as_string())
conn = None
try:
try:
nntp_host, nntp_port = Utils.nntpsplit(mlist.nntp_host)
conn = nntplib.NNTP(nntp_host, nntp_port,
readermode=True,
user=mm_cfg.NNTP_USERNAME,
password=mm_cfg.NNTP_PASSWORD)
conn.post(fp)
except nntplib.error_temp, e:
syslog('error',
'(NNTPDirect) NNTP error for list "%s": %s',
mlist.internal_name(), e)
except socket.error, e:
syslog('error',
'(NNTPDirect) socket error for list "%s": %s',
mlist.internal_name(), e)
finally:
if conn:
conn.quit()
def getPosts(self, params):
"""Get summary information about a newsgroup, and enqueue a task to fetch
some posts from that newsgroup."""
newsgroup = cgi.escape(params['newsgroup'])
cid = params['cid']
logging.info('got newsgroup: %s', newsgroup)
if not newsgroup:
self.response.out.write('Could not get newsgroup.')
return
try:
NNTP_SERVER = 'news.mixmin.net'
# get an nntp server object
s = nntplib.NNTP(NNTP_SERVER)
# get information about the given newsgroup
resp, count, first, last, name = s.group(newsgroup)
res_msg = ('resp, count, first, last, name: %s, %s, %s, %s, %s' %
(resp, count, first, last, name))
logging.info(res_msg)
# enqueue a task to fetch some posts from that newsgroup. Include the
# nntp object as part of the task payload.
defer(fetch_posts, s, newsgroup, last, cid)
except Exception, e:
logging.exception('something went wrong...')
self.render_json({'status': 'ERROR', 'error': str(e)})
return
def get_group(group):
s = nntplib.NNTP("news.epita.fr")
_, _, first, last, _ = s.group(group)
subjects = [(x[0], properly_decode_header(x[1]))
for x in s.xover(first, last)[1]]
subjects.reverse()
return flask.render_template('subjects.html', subjects=subjects)
def __init__(self, host, archive=None, from_archive=False):
self.host = host
self.archive = archive
self.from_archive = from_archive
if not self.from_archive:
self.handler = nntplib.NNTP(self.host)
def main():
with nntplib.NNTP(HOST) as n:
"""
- resp: server response if successful
- count: number of articles
- first: first article number
- last: last article number
- name: the group name
"""
print(n.getwelcome())
resp, count, first, last, name = n.group(GRNM)
print(f'Group {name} has {count} articles, range from {first} to {last}')
rng = f'{first}-{last}'
"""Process an XHDR command (optional server extension). Arguments:
- hdr: the header type (e.g. 'subject')
- str: an article nr, a message id, or a range nr1-nr2
- file: Filename string or file object to store the resu
lt in
Returns:
- resp: server response if successful
- list: list of (id, text) strings
text is the text of the requested header for that article.
"""
# 获取了从 first 到 last 的 from，subject， date字段信息
# 返回 （响应，lsit）
# list： [(序号1， 内容1)，。。。 ]
rsp, frm = n.xhdr('from', rng)
rsp, sub = n.xhdr('subject', rng)
rsp, date = n.xhdr('date', rng)
print(f'*** Find last article {last}: \nFrom: {frm[-1][1]}\nSubject: {sub[-1][1]}\nDate: {date[-1][1]}')
rsp, (num, nid, data) = n.body(last)
displayFirst20lines(data)
def main():
try:
n = nntplib.NNTP(HOST) #, user=USER, password=PASS)
except socket.gaierror, e:
print 'ERROR: cannot reach host "%s"' % HOST
print ' ("%s")' % eval(str(e))[1]
return
def _dispose(self, mlist, msg, msgdata):
# Get NNTP server connection information.
host = config.nntp.host.strip()
port = config.nntp.port.strip()
if len(port) == 0:
port = 119
else:
try:
port = int(port)
except (TypeError, ValueError):
log.exception('Bad [nntp]port value: {}'.format(port))
port = 119
# Make sure we have the most up-to-date state
if not msgdata.get('prepped'):
prepare_message(mlist, msg, msgdata)
# Flatten the message object, sticking it in a BytesIO object
fp = BytesIO(msg.as_bytes())
conn = None
try:
conn = nntplib.NNTP(host,
port,
readermode=True,
user=config.nntp.user,
password=config.nntp.password)
conn.post(fp)
except nntplib.NNTPTemporaryError:
# This could be a duplicate Message-ID for a message cross-posted
# to another group. See if we already munged the Message-ID.
mo = mcre.search(msg.get('message-id', 'n/a'))
if (mo and mo.group('listname') == mlist.list_name
and mo.group('hostname') == mlist.mail_host):
# This is our munged Message-ID. This must be a failure of the
# requeued message or a Message-ID we added in prepare_message.
# Get the original Message-ID or the one we added and log it.
log_message_id = msgdata.get('original_message-id',
msg.get('message-id', 'n/a'))
log.exception('{} NNTP error for {}'.format(
log_message_id, mlist.fqdn_listname))
else:
# This might be a duplicate Message-ID. Munge it and requeue
# the message, but save the original Message-ID for logging.
msgdata['original_message-id'] = msg.get('message-id', 'n/a')
del msg['message-id']
msg['Message-ID'] = email.utils.make_msgid(
mlist.list_name, mlist.mail_host)
return True
except socket.error:
log.exception('{} NNTP socket error for {}'.format(
msg.get('message-id', 'n/a'), mlist.fqdn_listname))
except Exception:
# Some other exception occurred, which we definitely did not
# expect, so set this message up for requeuing.
log.exception('{} NNTP unexpected exception for {}'.format(
msg.get('message-id', 'n/a'), mlist.fqdn_listname))
return True
finally:
if conn:
conn.quit()
return False
def main():
try:
n = nntplib.NNTP(HOST)
except socket.gaierror, e:
print 'ERROR: cannot reach host "%s"' % HOST
print ' ("%s")' % str(e)
return
def main():
try:
n = nntplib.NNTP(Host)
#, user = User, password = Pass
except socket.gaierror, e:
print 'ERROR: cannot reach host "%s"' % Host
print ' ("%s")' % eval(str(e))[1]
return
def list_groups():
nntpconn = nntplib.NNTP('news.gmane.io')
resp, lists = nntpconn.list()
for group, last, first, flag in lists:
print(group)
nntpconn.quit()
def has_newsserver(server):
import nntplib
try:
nntp = nntplib.NNTP(server, usenetrc=False)
nntp.quit()
return True
except nntplib.NNTPError:
return False
def get_message(group, num):
s = nntplib.NNTP('news.epita.fr')
s.group(group)
# Decode headers, and mask emails in it.
headers_set = ('subject', 'from', 'date')
headers = [
Header(name=nntp_to_header_name(line).capitalize(),
content=mask_email(nntp_to_header_content(line)))
for line in s.head(num)[3] if nntp_to_header_name(line) in headers_set
]
# Decode lines from their original charset to Unicode.
encoding = get_encoding(s, num)
if len(encoding) != 1:
encoding = 'utf-8'
else:
encoding = encoding[0]
# And mask email addresses in the same time.
lines = [
mask_email(line.decode(encoding, 'replace')) for line in s.body(num)[3]
]
# Tag lines to recognize quotes and signatures.
message = []
state = None
for line in lines:
# By default, do not change the state
new_state = state
if state == Line.SIGNATURE:
# After a signature mark, everything belongs to the signature
pass
elif state in (None, Line.QUOTE, Line.QUOTE_QUOTE):
if line[0:2] in Line.QUOTE_QUOTE_PREFIXES:
new_state = Line.QUOTE_QUOTE
elif line[0:1] in Line.QUOTE_PREFIXES:
new_state = Line.QUOTE
elif line == u'-- ':
new_state = Line.SIGNATURE
else:
new_state = None
else:
assert False, 'Wrong state: %s' % state
starts = None
if new_state != state:
if state is not None:
message[-1].ends = state
starts = new_state
message.append(Line(line, starts))
state = new_state
# Close any remaining tag.
if state != None:
message[-1].ends = state
return flask.render_template('message.html',
headers=headers,
message=message)
def connect_to_server(self):
self.established = False
try:
self.nntp = nntplib.NNTP(pylons.config['debexpo.nntp_server'])
except nntplib.NNTPError as e:
log.error("Connecting to NNTP server %s failed: %s" % (pylons.config['debexpo.nntp_server'], str(e)))
return
self.established = True
def get_co(host, port, ssl, user, password, timeout):
try:
if ssl:
co = nntplib.NNTP_SSL(host, port, user, password, timeout=timeout)
else:
co = nntplib.NNTP(host, port, user, password, timeout=timeout)
except Exception as err:
raise ConnectionError('Could not connect to the server, please, check '
'your connection ({}).'.format(err))
return co
def run(self):
for i in range(self.maxconn):
conn = nntplib.NNTP('127.0.0.1', port=7016)
self.conns.append(conn)
for x in range(1):
self.workers.append(gevent.spawn(self.download_article, i, conn))
result0 = gevent.joinall(self.workers)
self.result = ([worker.value for worker in self.workers])
for s in self.conns:
s.quit()
print("gevent shutdown done!")
def fetch(self, offset=DEFAULT_OFFSET):
"""Fetch articles posted on a news group.
This method fetches those messages or articles published
on a news group starting on the given offset.
:param offset: obtain messages from this offset
:returns: a generator of articles
"""
logger.info("Fetching articles of '%s' group on '%s' offset %s",
self.group, self.host, str(offset))
self._purge_cache_queue()
narts, iarts, tarts = (0, 0, 0)
# Connect with the server and select the given group
with nntplib.NNTP(self.host) as client:
_, _, first, last, _ = client.group(self.group)
if offset <= last:
first = max(first, offset)
_, overview = client.over((first, last))
else:
overview = []
tarts = len(overview)
logger.debug("Total number of articles to fetch: %s", tarts)
for article_id, _ in overview:
try:
article = self.__fetch_and_parse_article(
client, article_id)
except ParseError:
logger.warning("Error parsing %s article; skipping",
article_id)
iarts += 1
continue
except nntplib.NNTPTemporaryError as e:
logger.warning("Error '%s' fetching article %s; skipping",
e.response, article_id)
iarts += 1
continue
yield article
narts += 1
self._flush_cache_queue()
logger.info(
"Fetch process completed: %s/%s articles fetched; %s ignored",
narts, tarts, iarts)
def aam():
GROUP = "alt.anonymous.messages"
timeStamp, curTimeStamp = dhutils.getNewsTimestamp(gpg, dbpassphrase)
YYMMDD = time.strftime('%y%m%d', time.gmtime(timeStamp))
HHMMSS = time.strftime('%H%M%S', time.gmtime(timeStamp))
passphrases = dhutils.getListOfKeys(gpg, dbpassphrase)
# connect to server
server = nntplib.NNTP(NEWSSERVER, NEWSPORT)
server.newnews(GROUP, YYMMDD, HHMMSS, '.newnews')
with open('.newnews', "r") as f:
ids = f.read().splitlines()
for msg_id in ids:
try:
resp, number, message_id, text = server.article(msg_id)
except (nntplib.error_temp, nntplib.error_perm):
pass # no such message (maybe it was deleted?)
text = string.join(text, "\n")
message = email.message_from_string(text)
match = False
for passphrase in passphrases:
for label, item in message.items():
if label == 'Subject':
match = hsub.check(passphrase[2][:16], item)
#if match, decrypt
if match:
print '\nMail for: ' + passphrase[
0] + ' from ' + passphrase[1]
msg = gpg.decrypt(message.as_string(),
passphrase=passphrase[2],
always_trust=True)
if not msg:
print 'Bad shared secret!'
sys.exit(1)
print '\n' + unicode(msg)
with open('message_' + message_id[1:6] + '.txt',
"w") as f:
f.write('X-encoDHer-Route: ' + passphrase[1] +
'->' + passphrase[0] + '\n')
f.write(message.as_string() + '\n')
print 'encrypted message stored in message_' + message_id[
1:6] + '.txt'
dhutils.setNewsTimestamp(curTimeStamp, gpg, dbpassphrase)
print 'End of messages.'
def test():
s = nntplib.NNTP('newsvr')
the_newsgroup = "comp.lang.python"
now = time.time()
two_days = 2 * 24 * 60 * 60
article_info = get_articles_younger_than(s, the_newsgroup, two_days)
for i in article_info:
#pprint.pprint( article_info )
#pprint.pprint( s.head( article_info[0][0] ) )
print get_google_url_for_messageid(i[1])
s.quit()