def _tunnel(self):
connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host,
self._tunnel_port)
connect_bytes = connect_str.encode("ascii")
self.send(connect_bytes)
for header, value in self._tunnel_headers.items():
header_str = "%s: %s\r\n" % (header, value)
header_bytes = header_str.encode("latin-1")
self.send(header_bytes)
self.send(b'\r\n')
response = self.response_class(self.sock, method=self._method)
(version, code, message) = response._read_status()
if code != http.HTTPStatus.OK:
self.close()
raise OSError("Tunnel connection failed: %d %s" %
(code, message.strip()))
while True:
line = response.fp.readline(_MAXLINE + 1)
if len(line) > _MAXLINE:
raise LineTooLong("header line")
if not line:
# for sites which EOF without sending a trailer
break
if line in (b'\r\n', b'\n', b''):
break
if self.debuglevel > 0:
print('header:', line.decode())
def _tunnel(self):
connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host,
self._tunnel_port)
connect_bytes = connect_str.encode("ascii")
self.send(connect_bytes)
for header, value in self._tunnel_headers.items():
header_str = "%s: %s\r\n" % (header, value)
header_bytes = header_str.encode("latin-1")
self.send(header_bytes)
self.send(b'\r\n')
response = self.response_class(self.sock, method=self._method)
(version, code, message) = response._read_status()
if code != http.HTTPStatus.OK:
self.close()
raise OSError("Tunnel connection failed: %d %s" % (code,
message.strip()))
while True:
line = response.fp.readline(_MAXLINE + 1)
if len(line) > _MAXLINE:
raise LineTooLong("header line")
if not line:
# for sites which EOF without sending a trailer
break
if line in (b'\r\n', b'\n', b''):
break
if self.debuglevel > 0:
print('header:', line.decode())
def _tunnel(self):
self._set_hostport(self._tunnel_host, self._tunnel_port)
connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port)
connect_bytes = connect_str.encode("ascii")
self.send(connect_bytes)
for header, value in self._tunnel_headers.items():
header_str = "%s: %s\r\n" % (header, value)
header_bytes = header_str.encode("latin-1")
self.send(header_bytes)
self.send(b"\r\n")
response = self.response_class(self.sock, method=self._method)
(version, code, message) = response._read_status()
if code != 200:
self.close()
raise socket.error("Tunnel connection failed: %d %s" %
(code, message.strip()))
while True:
line = response.fp.readline(_MAXLINE + 1)
if len(line) > _MAXLINE:
raise LineTooLong("header line")
if not line:
# for sites which EOF without sending a trailer
break
if line in (b"\r\n", b"\n", b""):
break
def _tunnel(self):
self._set_hostport(self._tunnel_host, self._tunnel_port)
connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port)
connect_bytes = connect_str.encode("ascii")
self.send(connect_bytes)
for header, value in self._tunnel_headers.items():
header_str = "%s: %s\r\n" % (header, value)
header_bytes = header_str.encode("latin-1")
self.send(header_bytes)
self.send(b"\r\n")
response = self.response_class(self.sock, method=self._method)
(version, code, message) = response._read_status()
if code != 200:
self.close()
raise socket.error("Tunnel connection failed: %d %s" % (code, message.strip()))
while True:
line = response.fp.readline(_MAXLINE + 1)
if len(line) > _MAXLINE:
raise LineTooLong("header line")
if not line:
# for sites which EOF without sending a trailer
break
if line in (b"\r\n", b"\n", b""):
break
def _log(self, message, *args, **kwargs):# rec=0, verbosity=1):
"""
Log a message using :mod:`syslog` as well as :attr:`sys.stdout`.
:param message: The pre-formatted log message.
:param rec: Optional recursion level used to indent messages.
:param verbosity: Minimal verbosity required to display this message.
:returns: :const:`None`
"""
verbosity = kwargs.get('verbosity', 1)
rec = kwargs.get('rec', 0)
if self._conf.verbose and self._conf.verbosity >= verbosity:
format_args = []
for a in args:
if isinstance(a, unicode):
format_args.append(a.encode('UTF-8'))
else:
format_args.append(a)
if isinstance(message, unicode):
message = message.encode('UTF-8')
message = '{0}{1}'.format(' '*rec, message.format(*format_args))
if verbosity < 2:
if message.strip().startswith('!!!'):
self._logger.error(message)
else:
self._logger.info(message)
else:
if message.strip().startswith('!!!'):
self._logger.warning(message)
else:
self._logger.debug(message)
def _log(self, message, *args, **kwargs): # rec=0, verbosity=1):
"""
Log a message using :mod:`syslog` as well as :attr:`sys.stdout`.
:param message: The pre-formatted log message.
:param rec: Optional recursion level used to indent messages.
:param verbosity: Minimal verbosity required to display this message.
:returns: :const:`None`
"""
verbosity = kwargs.get('verbosity', 1)
rec = kwargs.get('rec', 0)
if self._conf.verbose and self._conf.verbosity >= verbosity:
format_args = []
for a in args:
if isinstance(a, unicode):
format_args.append(a.encode('UTF-8'))
else:
format_args.append(a)
if isinstance(message, unicode):
message = message.encode('UTF-8')
message = '{0}{1}'.format(' ' * rec, message.format(*format_args))
if verbosity < 2:
if message.strip().startswith('!!!'):
self._logger.error(message)
else:
self._logger.info(message)
else:
if message.strip().startswith('!!!'):
self._logger.warning(message)
else:
self._logger.debug(message)
def _tunnel(self):
self._set_hostport(self._tunnel_host, self._tunnel_port)
connect_str = "CONNECT %s:%d HTTP/1.0\r\n\r\n" %(self.host, self.port)
connect_bytes = connect_str.encode("ascii")
self.send(connect_bytes)
response = self.response_class(self.sock, strict = self.strict,
method= self._method)
(version, code, message) = response._read_status()
if code != 200:
self.close()
raise socket.error("Tunnel connection failed: %d %s" % (code,
message.strip()))
while True:
line = response.fp.readline()
if line == b'\r\n':
break
def _tunnel(self):
self._set_hostport(self._tunnel_host, self._tunnel_port)
connect_str = 'CONNECT %s:%d HTTP/1.0\r\n' % (self.host, self.port)
connect_bytes = connect_str.encode('ascii')
self.send(connect_bytes)
for (header, value) in self._tunnel_headers.items():
header_str = '%s: %s\r\n' % (header, value)
header_bytes = header_str.encode('latin-1')
self.send(header_bytes)
self.send(b'\r\n')
response = self.response_class(self.sock, method=self._method)
(version, code, message) = response._read_status()
if code != 200:
self.close()
raise socket.error('Tunnel connection failed: %d %s' % (code, message.strip()))
while True:
line = response.fp.readline(_MAXLINE + 1)
if len(line) > _MAXLINE:
raise LineTooLong('header line')
if not line:
break
if line in (b'\r\n', b'\n', b''):
break
def _tunnel(self):
self._set_hostport(self._tunnel_host, self._tunnel_port)
connect_str = 'CONNECT %s:%d HTTP/1.0\r\n' % (self.host, self.port)
connect_bytes = connect_str.encode('ascii')
self.send(connect_bytes)
for (header, value) in self._tunnel_headers.items():
header_str = '%s: %s\r\n' % (header, value)
header_bytes = header_str.encode('latin-1')
self.send(header_bytes)
self.send(b'\r\n')
response = self.response_class(self.sock, method=self._method)
(version, code, message) = response._read_status()
if code != 200:
self.close()
raise socket.error('Tunnel connection failed: %d %s' %
(code, message.strip()))
while True:
line = response.fp.readline(_MAXLINE + 1)
if len(line) > _MAXLINE:
raise LineTooLong('header line')
if not line:
break
if line in (b'\r\n', b'\n', b''):
break
def news2mail(self, fobj=sys.stdin):
"""
This method provides a drop-in-replacement to news2mail.pl used by INN_.
It expects the same data on :attr:`sys.stdin` and uses the same
config entry as news2mail.pl.
.. note::
There is no need to import and call this method directly.
SynFu provides the wrapper script :command:`synfu-news2mail` (see :ref:`synfu-news2mail`) for this job.
"""
"""
This is based on *partial* documentation of news2mail.pl and INN
On entry :attr:`sys.stdin` contains a set of::
@token@ list-ids
pairs for each article that needs to be mailed.
According to INN we need to call '*sm*' to get the actual message.
(which we mangle and filter and then pipe through to sendmail)
"""
ltok = re.compile(r'\s+')
self._log('--- begin')
line = fobj.readline()
while line:
line = line.strip()
if not line:
self._log('--- received an empty line on STDIN - exiting.')
break
(token, names) = ltok.split(line.strip(), 1)
addrs = {}
self._log('--- processing LTOK = \'{0}\'', line, verbosity=2)
# XXX: this could be done !!FASTER!!
for name in names.split(','):
name = name.strip()
for e in self._conf.filters:
if not 'nntp' in e or \
not 'from' in e:
continue
if e['nntp'] == name:
sender = self._conf.default_sender
if 'sender' in e:
sender = e['sender']
if not sender in addrs:
addrs[sender] = set()
if 'broken_auth' in e:
sender_is_from = True
else:
sender_is_from = False
addrs[sender].update([
(e['from'], sender_is_from),
])
break
if not addrs:
self._log('!!! No recipients for LTOK = \'{0}\'', line)
line = fobj.readline()
continue
self._log('--- addrs: {0}', addrs)
# now we have one message-token and the lists we should mail it to
# let's collect and process the actual messages as well as send them.
sm = subprocess.Popen('{0} -q {1}'.format(self._conf.inn_sm,
token),
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=sys.stderr)
message = sm.communicate()[0]
if message.strip():
mm = email.email.message_from_string(message)
mm = self._apply_blacklist(mm, 'news2mail', 0)
if not mm:
self._log('--- Message was dropped by blacklist')
line = fobj.readline()
continue
tag = self._find_list_tag(mm)
mm._headers = self._filter_headers(tag, mm._headers)
for sender in addrs:
try:
mm.replace_header(
'To', ','.join(x[0] for x in addrs[sender]))
except KeyError:
mm._headers.append(
('To', ','.join(x[0] for x in addrs[sender])))
try:
mm.replace_header('Sender', sender)
except KeyError:
mm._headers.append(('Sender', sender))
if any(x[1] for x in addrs[sender]):
# at least one recipient list requires From == Sender
# Sieht spannend aus, funktioniert aber vermutlich.
msg_from = ','.join(
x for x in [mm.get('From', None), sender] if x)
try:
mm.replace_header('From', msg_from)
except KeyError:
mm._headers.append(('From', msg_from))
self._log(
'--- at least one recipient requires From == Sender',
verbosity=3)
self._log('--- therefore I\'m forcing "From:" to {0}',
msg_from,
verbosity=3)
else:
msg_from = mm.get('From', sender)
self._log('--- this is a clean list', verbosity=3)
self._log(
'--- therefore I\'m keepfing "From:" set to {0}',
msg_from,
verbosity=3)
for i in xrange(len(mm._headers) - 1, -1, -1):
(k, v) = mm._headers[i]
if k.lower() == 'newsgroups':
mm._headers.remove((k, v))
mm._headers.append(('X-Newsgroups', v))
elif k.lower() == 'followup-to':
mm._headers.remove((k, v))
mm._headers.append(('X-Followup-To', v))
self._log('--- Save X-Followup-To "{0}"',
v,
verbosity=2)
v = v.strip() # should be one newsgroup
for e in self._conf.filters:
if not 'nntp' in e or \
not 'from' in e:
continue
if e['nntp'] == v and not mm.get(
'Mail-Followup-To'):
mm._headers.append(
('Mail-Followup-To', e['from']))
self._log(
'--- Set Mail-Followup-To to "{0}"',
e['from'],
verbosity=2)
if self._conf.use_path_marker:
path = mm.get('Path', None)
if path is None:
mm._headers.append(
('Path', self._conf.path_marker))
self._log('--- adding path marker "{0}"'.format(
self._conf.path_marker))
else:
if not self._conf.path_marker in path.split('!'):
path.append('!{0}'.format(
self._conf.path_marker))
mm.replace_header('Path', path)
self._log(
'--- adding path marker to existing Path "{0}"'
.format(self._conf.path_marker))
else:
self._log(
'!!! Path-Header already contains a valid path_marker!?'
)
mm.add_header('X-SynFU-PostFilter',
PostFilter.NOTICE,
version=PostFilter.VERSION)
sendmail = subprocess.Popen(
self._conf.news2mail_cmd.format(
{
'FROM': msg_from,
'SENDER': sender,
'HOST': self._conf.inn_host
}, ' '.join(x[0] for x in addrs[sender])),
shell=True,
stdin=subprocess.PIPE,
stdout=sys.stdout,
stderr=sys.stderr)
sendmail.communicate(str(mm))
line = fobj.readline()
self._log('--- end')
return 0
def news2mail(self, fobj=sys.stdin):
"""
This method provides a drop-in-replacement to news2mail.pl used by INN_.
It expects the same data on :attr:`sys.stdin` and uses the same
config entry as news2mail.pl.
.. note::
There is no need to import and call this method directly.
SynFu provides the wrapper script :command:`synfu-news2mail` (see :ref:`synfu-news2mail`) for this job.
"""
"""
This is based on *partial* documentation of news2mail.pl and INN
On entry :attr:`sys.stdin` contains a set of::
@token@ list-ids
pairs for each article that needs to be mailed.
According to INN we need to call '*sm*' to get the actual message.
(which we mangle and filter and then pipe through to sendmail)
"""
ltok = re.compile(r'\s+')
self._log('--- begin')
line = fobj.readline()
while line:
line = line.strip()
if not line:
self._log('--- received an empty line on STDIN - exiting.');
break
(token, names) = ltok.split(line.strip(), 1)
addrs = {}
self._log('--- processing LTOK = \'{0}\'', line, verbosity=2)
# XXX: this could be done !!FASTER!!
for name in names.split(','):
name = name.strip()
for e in self._conf.filters:
if not 'nntp' in e or \
not 'from' in e:
continue
if e['nntp'] == name:
sender = self._conf.default_sender
if 'sender' in e:
sender = e['sender']
if not sender in addrs:
addrs[sender] = set()
if 'broken_auth' in e:
sender_is_from = True
else:
sender_is_from = False
addrs[sender].update([(e['from'], sender_is_from),])
break
if not addrs:
self._log('!!! No recipients for LTOK = \'{0}\'', line)
line = fobj.readline()
continue
self._log('--- addrs: {0}', addrs)
# now we have one message-token and the lists we should mail it to
# let's collect and process the actual messages as well as send them.
sm = subprocess.Popen('{0} -q {1}'.format(self._conf.inn_sm, token),
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=sys.stderr)
message = sm.communicate()[0]
if message.strip():
mm = email.email.message_from_string(message)
mm = self._apply_blacklist(mm, 'news2mail', 0)
if not mm:
self._log('--- Message was dropped by blacklist')
line = fobj.readline()
continue
tag = self._find_list_tag(mm)
mm._headers = self._filter_headers(tag, mm._headers)
for sender in addrs:
try:
mm.replace_header('To', ','.join(x[0] for x in addrs[sender]))
except KeyError:
mm._headers.append(('To', ','.join(x[0] for x in addrs[sender])))
try:
mm.replace_header('Sender', sender)
except KeyError:
mm._headers.append(('Sender', sender))
if any(x[1] for x in addrs[sender]):
# at least one recipient list requires From == Sender
# Sieht spannend aus, funktioniert aber vermutlich.
msg_from = ','.join(x for x in [mm.get('From', None), sender] if x)
try:
mm.replace_header('From', msg_from)
except KeyError:
mm._headers.append(('From', msg_from))
self._log('--- at least one recipient requires From == Sender', verbosity=3)
self._log('--- therefore I\'m forcing "From:" to {0}', msg_from, verbosity=3)
else:
msg_from = mm.get('From', sender)
self._log('--- this is a clean list', verbosity=3)
self._log('--- therefore I\'m keepfing "From:" set to {0}', msg_from, verbosity=3)
for i in xrange(len(mm._headers) - 1, -1, -1):
(k, v) = mm._headers[i]
if k.lower() == 'newsgroups':
mm._headers.remove((k, v))
mm._headers.append(('X-Newsgroups', v))
elif k.lower() == 'followup-to':
mm._headers.remove((k, v))
mm._headers.append(('X-Followup-To', v))
self._log('--- Save X-Followup-To "{0}"', v, verbosity=2)
v = v.strip() # should be one newsgroup
for e in self._conf.filters:
if not 'nntp' in e or \
not 'from' in e:
continue
if e['nntp'] == v and not mm.get('Mail-Followup-To'):
mm._headers.append(('Mail-Followup-To', e['from']))
self._log('--- Set Mail-Followup-To to "{0}"', e['from'], verbosity=2)
if self._conf.use_path_marker:
path = mm.get('Path', None)
if path is None:
mm._headers.append(('Path', self._conf.path_marker))
self._log('--- adding path marker "{0}"'.format(self._conf.path_marker))
else:
if not self._conf.path_marker in path.split('!'):
path.append('!{0}'.format(self._conf.path_marker))
mm.replace_header('Path', path)
self._log('--- adding path marker to existing Path "{0}"'.format(self._conf.path_marker))
else:
self._log('!!! Path-Header already contains a valid path_marker!?')
mm.add_header('X-SynFU-PostFilter',
PostFilter.NOTICE, version=PostFilter.VERSION)
sendmail = subprocess.Popen(self._conf.news2mail_cmd.format(
{
'FROM' : msg_from,
'SENDER' : sender,
'HOST' : self._conf.inn_host
},
' '.join(x[0] for x in addrs[sender])),
shell=True,
stdin=subprocess.PIPE,
stdout=sys.stdout,
stderr=sys.stderr)
sendmail.communicate(str(mm))
line = fobj.readline()
self._log('--- end')
return 0
