def format(self, events, encoding="utf-8"):
parts = list()
data = templates.Template.format(self, parts, events)
parsed = message_from_string(data.encode(encoding))
charset = Charset(encoding)
charset.header_encoding = QP
msg = MIMEMultipart()
msg.set_charset(charset)
for key, value in msg.items():
del parsed[key]
for key, value in parsed.items():
msg[key] = value
for encoded in ["Subject", "Comment"]:
if encoded not in msg:
continue
value = charset.header_encode(msg[encoded])
del msg[encoded]
msg[encoded] = value
del msg["Content-Transfer-Encoding"]
msg["Content-Transfer-Encoding"] = "7bit"
msg.attach(MIMEText(parsed.get_payload(), "plain", encoding))
for part in parts:
msg.attach(part)
return msg
def format(self, events, encoding="utf-8"):
from email import message_from_string
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.charset import Charset, QP
from email.utils import formatdate, make_msgid
parts = list()
data = templates.Template.format(self, parts, events)
parsed = message_from_string(data.encode(encoding))
charset = Charset(encoding)
charset.header_encoding = QP
msg = MIMEMultipart()
msg.set_charset(charset)
for key, value in msg.items():
del parsed[key]
for key, value in parsed.items():
msg[key] = value
for encoded in ["Subject", "Comment"]:
if encoded not in msg:
continue
value = charset.header_encode(msg[encoded])
del msg[encoded]
msg[encoded] = value
del msg['Content-Transfer-Encoding']
msg['Content-Transfer-Encoding'] = '7bit'
msg.attach(MIMEText(parsed.get_payload(), "plain", encoding))
for part in parts:
msg.attach(part)
return msg
class OpenOCRService(object):
"""
This is the python script that can be incorporated
in your python applications to connect with the openocr
services run by open-ocr docker. This is the interpretation
of the upload-local-file.sh which is a shell script to
interact with the service using curl. The service expects
a multipart/related Content-Type.
This service just expects the image path in your local machine
"""
TESSERACT_OCR_SERVICE_URL = 'http://openocr:9292/ocr-file-upload'
def __init__(self, image_path):
self.image_path = image_path
self.related = MIMEMultipart('related')
self.image_format = self.image_path.split('/')[-1].split('.')[-1]
self.set_image_payload()
self.set_data_payload()
def set_image_payload(self):
submission = MIMEImage('image', '{}'.format(self.image_format))
submission.set_payload(open(self.image_path, 'rb').read())
self.related.attach(submission)
def set_data_payload(self):
data = MIMEText('application', 'json')
data.set_payload(json.dumps({"engine": "tesseract", "preprocessors" : ["stroke-width-transform"]}))
self.related.attach(data)
def get_body(self):
return self.related.as_string().split('\n\n', 1)[1]
def get_headers(self):
return dict(self.related.items())
def perform_ocr(self):
body = self.get_body()
headers = self.get_headers()
response = requests.post(self.TESSERACT_OCR_SERVICE_URL, data = body, headers = headers)
return response
def get_ocr_text(self):
response = self.perform_ocr()
return response.content
def __call__(self, client: Client, request_xml: str) -> str:
if len(self.attachments) > 0:
related = MIMEMultipart('related')
# add the soap message portion
xml = MIMEText("text", "xml")
xml.set_payload(request_xml)
related.attach(xml)
for attachment in self.attachments:
self.__add_related_item(related, attachment)
body = related.as_string().split('\n\n', 1)[1]
client.headers.update(dict(related.items()))
return body
else:
return request_xml
def create_email():
"""
Create message object using template and random data
returns:
email.Message object and the email as a standard dictionary
"""
sender = fake.email() # pylint: disable=no-member
recipient = fake.email() # pylint: disable=no-member
cc = [fake.email() for _ in range(random.randint(0, 2))] # pylint: disable=no-member
bcc = [fake.email() for _ in range(random.randint(0, 2))] # pylint: disable=no-member
the_time = datetime.now()
received = 'from ' + fake.hostname() + ' (' + fake.ipv4_public() # pylint: disable=no-member
received += ')\r\n' + 'by ' + fake.domain_word() + '.' # pylint: disable=no-member
received += fake.free_email_domain() + ' with ' # pylint: disable=no-member
received += EMAIL_PROTOCOLS[random.randint(0, len(EMAIL_PROTOCOLS) - 1)]
received += '; ' + the_time.strftime('%c')
msg = MIMEMultipart('alternative')
msg['Subject'] = fake.sentence() # pylint: disable=no-member
msg['From'] = sender
msg['Reply-To'] = sender
msg['To'] = recipient
msg['Message-ID'] = fake.uuid4() # pylint: disable=no-member
msg['CC'] = ', '.join(cc) if cc else ''
msg['BCC'] = ', '.join(bcc) if bcc else ''
msg['User-Agent'] = fake.user_agent() # pylint: disable=no-member
msg['Date'] = the_time.strftime("%Y-%m-%dT%H:%M:%SZ")
msg['Received'] = received
plaintext = create_content()
html = inject_content_into_template(plaintext)
part1 = MIMEText(plaintext, 'plain')
part2 = MIMEText(html, 'html')
msg.attach(part1)
msg.attach(part2)
email_object = {}
for key, val in msg.items():
email_object[key] = val
email_object['Text'] = plaintext
email_object['Body'] = html
return msg, email_object
def post_image(url, image, omexml, **kwargs):
'''Post an image to the webserver via the writeimage web interface
url - file or http url of the file to be created
image - an N-D image, scaled appropriately for the OME data type
omexml - the OME xml metadata for the image
Accepts post parameters via keyword arguments, for example,
post_image(path, compression="LZW", series=3, index=1, channel=2)
'''
message = MIMEMultipart()
message.set_type("multipart/form-data")
message.add_header("Connection","close")
d = dict([(key, MIMEText(value.encode("utf-8"), "plain", "utf-8"))
for key, value in kwargs.iteritems()])
d["omexml"] = MIMEText(omexml.encode("utf-8"), "xml", "utf-8")
d["image"] = MIMEApplication(encode_image(image))
d["url"] = MIMEText(url)
for key, value in d.iteritems():
value.add_header("Content-Disposition", "form-data", name=key)
message.attach(value)
conn = connect()
try:
body = message.as_string()
# oooo - translate line-feeds w/o carriage returns into cr/lf
# The generator uses print to write the message, how bad.
# This keeps us from being able to encode in binary too.
#
body = "\r\n".join(re.split("(?<!\\r)\\n", body))
conn.request("POST", "/writeimage", body, dict(message.items()))
response = conn.getresponse()
if response.status != httplib.OK:
raise HTTPError(response, "Image server failed to write image. URL="+url)
finally:
conn.close()
def upload(self, chunk, data, **kwargs):
"""
Overridden to perform uploads.
Google is the most problematic of the clouds. Their API is pretty
horrible to work with for the following reasons:
They are _very_ particular in the formatting of requests, and they
require atypical formatting, so not fun.
Something that works one day will stop working the next. In particular
what MIME types they accept. In fact most of my problems have been
related to MIME types.
"""
assert isinstance(chunk, Chunk), 'must be chunk instance'
try:
parent_id = self.storage.attrs.get('root.id')
except ValueError:
parent_id = None
attrs = {
'mimeType': 'text/plain',
'title': chunk.uid,
'description': 'Cloudstrype chunk',
}
if parent_id:
attrs['parents'] = [{'id': parent_id}]
# Google wants multipart/related, which requests does not do by default
# so we craft the multipart request body using MIME tools.
related = MIMEMultipart('related')
jsonpart = MIMEBase('application', 'json', charset='utf-8')
jsonpart.set_payload(json.dumps(attrs))
related.attach(jsonpart)
chunkpart = MIMEBase('application', 'octet-stream')
chunkpart.add_header('Content-Transfer-Encoding', 'base64')
chunkpart.set_payload(b64encode(data))
related.attach(chunkpart)
# Get the body, discarding the headers, then get the headers as a dict
# allowing requests to handle the headers.
body = related.as_bytes().split(b'\n\n', 1)[1]
headers = dict(related.items())
method, url = self.UPLOAD_URL
url += '?uploadType=multipart'
r = self.request(method,
url,
chunk,
data=body,
headers=headers,
**kwargs)
if not 199 < r.status_code < 300:
raise HTTPError(response=r)
attrs = r.json()
# Store the file ID provided by Google into the attribute store of
# ChunkStorage
try:
return {'file.id': attrs['id']}
except KeyError as e:
LOGGER.error('key "id" not in response "%s"', attrs)
raise
def apply_mtom(headers, envelope, params, paramvals):
'''Apply MTOM to a SOAP envelope, separating attachments into a
MIME multipart message.
Returns a tuple of length 2 with dictionary of headers and string of body
that can be sent with HTTPConnection
References:
XOP http://www.w3.org/TR/xop10/
MTOM http://www.w3.org/TR/soap12-mtom/
http://www.w3.org/Submission/soap11mtom10/
:param headers Headers dictionary of the SOAP message that would
originally be sent.
:param envelope Iterable containing SOAP envelope string that would have
originally been sent.
:param params params attribute from the Message object used for the SOAP
:param paramvals values of the params, passed to Message.to_parent_element
'''
# grab the XML element of the message in the SOAP body
envelope = ''.join(envelope)
soaptree = etree.fromstring(envelope)
soapbody = soaptree.find("{%s}Body" % _ns_soap_env)
message = None
for child in list(soapbody):
if child.tag == ("{%s}Fault" % _ns_soap_env):
return (headers, envelope)
else:
message = child
break
# Get additional parameters from original Content-Type
ctarray = []
for n, v in headers.items():
if n.lower() == 'content-type':
ctarray = v.split(';')
break
roottype = ctarray[0].strip()
rootparams = {}
for ctparam in ctarray[1:]:
n, v = ctparam.strip().split('=')
rootparams[n] = v.strip("\"'")
# Set up initial MIME parts.
mtompkg = MIMEMultipart('related', boundary='?//<><>spyne_MIME_boundary<>')
rootpkg = MIMEApplication(envelope, 'xop+xml', encode_7or8bit)
# Set up multipart headers.
del(mtompkg['mime-version'])
mtompkg.set_param('start-info', roottype)
mtompkg.set_param('start', '<spyneEnvelope>')
if 'SOAPAction' in headers:
mtompkg.add_header('SOAPAction', headers.get('SOAPAction'))
# Set up root SOAP part headers.
del(rootpkg['mime-version'])
rootpkg.add_header('Content-ID', '<spyneEnvelope>')
for n, v in rootparams.items():
rootpkg.set_param(n, v)
rootpkg.set_param('type', roottype)
mtompkg.attach(rootpkg)
# Extract attachments from SOAP envelope.
for i in range(len(params)):
name, typ = params[i]
if typ in (ByteArray, Attachment):
id = "spyneAttachment_%s" % (len(mtompkg.get_payload()), )
param = message[i]
param.text = ""
incl = etree.SubElement(param, "{%s}Include" % _ns_xop)
incl.attrib["href"] = "cid:%s" % id
if paramvals[i].fileName and not paramvals[i].data:
paramvals[i].load_from_file()
if type == Attachment:
data = paramvals[i].data
else:
data = ''.join(paramvals[i])
attachment = None
attachment = MIMEApplication(data, _encoder=encode_7or8bit)
del(attachment['mime-version'])
attachment.add_header('Content-ID', '<%s>' % (id, ))
mtompkg.attach(attachment)
# Update SOAP envelope.
rootpkg.set_payload(etree.tostring(soaptree))
# extract body string from MIMEMultipart message
bound = '--%s' % (mtompkg.get_boundary(), )
marray = mtompkg.as_string().split(bound)
mtombody = bound
mtombody += bound.join(marray[1:])
# set Content-Length
mtompkg.add_header("Content-Length", str(len(mtombody)))
# extract dictionary of headers from MIMEMultipart message
mtomheaders = {}
for name, value in mtompkg.items():
mtomheaders[name] = value
if len(mtompkg.get_payload()) <= 1:
return (headers, envelope)
return (mtomheaders, [mtombody])
def run(self, email, *args, **kwargs):
self.log.info("Making email letter")
headers = email['headers']
body = email['body']
splitFrom = headers['from']['email'].split('@')
from_host = splitFrom[1]
replyTo = '%s+%s@%s' % (splitFrom[0], headers['guid'], from_host)
unsubscribe = '%s+unsubscribe_%s@%s' % (
splitFrom[0], headers['guid'], from_host)
self.log.info("Begin real sending to %s" % headers['to']['email'])
mixed = MIMEMultipart('mixed')
related = MIMEMultipart('related')
html_part = MIMEText(body['html'], 'html', 'utf-8')
text_part = MIMEText(body['plain'], 'plain', 'utf-8')
# Create message container - the correct MIME type is
# multipart/alternative.
alternative = MIMEMultipart(
'alternative', None, [text_part, html_part])
mixed['From'] = formataddr((str(Header(
headers['from']['name'],
'utf-8')),
headers['from']['email']))
mixed['To'] = formataddr((str(Header(
headers['to']['name'],
'utf-8')),
headers['to']['email']))
mixed['Subject'] = headers['Subject']
mixed['Reply-To'] = replyTo
Message_Id = make_msgid(headers['guid'])
mixed['Message-ID'] = Message_Id
mixed['Date'] = formatdate(localtime=True)
mixed['Precedence'] = 'bulk'
mixed['Sensitivity'] = 'Personal'
mixed['Content-Language'] = 'ru'
mixed['List-Unsubscribe'] = '<mailto:%s>' % unsubscribe
mixed["X-Sender"] = formataddr((str(Header(
headers['from']['name'],
'utf-8')),
headers['from']['email']))
mixed['List-id'] = headers['guid']
mixed["X-Mailer"] = "smtplib"
mixed["X-Priority"] = "3" # 1 UrgentMessage, 3 Normal
mixed["Importance"] = "3"
for element in body['attachment']:
filename = element['filename']
try:
if element['mimetype']:
maintype, subtype = element['mimetype'].split('/', 1)
else:
maintype, subtype = ('application', 'octet-stream')
except ValueError:
maintype, subtype = ('application', 'octet-stream')
part = MIMEBase(maintype, subtype)
part.set_payload(element['content'])
# Encode the payload using Base64
encode_base64(part)
# Set the filename parameter
part.add_header('Content-Disposition', 'attachment',
filename="%s" % Header(filename, 'utf-8'))
mixed.attach(part)
# If message have inline image - it must be attached
# to related section
related.attach(alternative)
mixed.attach(related)
config_path = path.dirname(path.abspath(__file__))
dkim_exist = False
try:
key_path = '%s/config/dkim.%s._.key' % (config_path, from_host)
f = open(key_path)
f.close()
dkim_exist = True
except IOError:
try:
from_host = 'babypages.ru'
mixed["Envelope-From"] = 'robot@%s' % from_host
key_path = '%s/config/dkim.%s._.key' % (config_path, from_host)
f = open(key_path)
f.close()
dkim_exist = True
except IOError:
pass
if dkim_exist:
with open(key_path) as f:
selector = '_'
privkey = f.read()
include_headers = frozenset(
self.valid_include_headers).intersection(mixed.items())
try:
dkim_header = dkim.sign(
mixed.as_string(), selector, from_host, privkey,
canonicalize=(dkim.Relaxed, dkim.Relaxed),
include_headers=include_headers)
dkim_header = Parser().parsestr(dkim_header)
dkim_value = dkim_header['DKIM-Signature']
mixed['dkim-signature'] = dkim_value
except dkim.KeyFormatError as e:
self.log.exception(e)
self.log.info("Message lenght: %d" % len(mixed.as_string()),
exc_info=True)
result = (mixed.as_string(), headers['to']['email'],
headers['from']['email'])
return result
def test_add_attachement_no_ctype_encoding_compressed(self):
config = EmailConfiguration()
sender = MockEmailSender(config, mock_sender=MockSMTPServer("127.0.0.1", 80))
msg = MIMEMultipart()
attachment = os.path.dirname(__file__) + os.sep + "something.zip"
sender._add_attachement(msg, attachment, ctype=None, encoding="utf-8")
self.assertEquals([('Content-Type', 'multipart/mixed'), ('MIME-Version', '1.0')], msg.items())
self.assertEquals(1, len(msg.get_payload()))
def apply_mtom(headers, envelope, params, paramvals):
'''
Apply MTOM to a SOAP envelope, separating attachments into a
MIME multipart message.
References:
XOP http://www.w3.org/TR/xop10/
MTOM http://www.w3.org/TR/soap12-mtom/
http://www.w3.org/Submission/soap11mtom10/
@param headers Headers dictionary of the SOAP message that would
originally be sent.
@param envelope SOAP envelope string that would have originally been sent.
@param params params attribute from the Message object used for the SOAP
@param paramvals values of the params, passed to Message.to_xml
@return tuple of length 2 with dictionary of headers and
string of body that can be sent with HTTPConnection
'''
# grab the XML element of the message in the SOAP body
soapmsg = StringIO(envelope)
soaptree = ElementTree.parse(soapmsg)
soapns = soaptree.getroot().tag.split('}')[0].strip('{')
soapbody = soaptree.getroot().find("{%s}Body" % soapns)
message = None
for child in list(soapbody):
if child.tag != "%sFault" % (soapns, ):
message = child
break
# Get additional parameters from original Content-Type
ctarray = []
for n, v in headers.items():
if n.lower() == 'content-type':
ctarray = v.split(';')
break
roottype = ctarray[0].strip()
rootparams = {}
for ctparam in ctarray[1:]:
n, v = ctparam.strip().split('=')
rootparams[n] = v.strip("\"'")
# Set up initial MIME parts
mtompkg = MIMEMultipart('related',
boundary='?//<><>soaplib_MIME_boundary<>')
rootpkg = None
try:
rootpkg = MIMEApplication(envelope, 'xop+xml', encode_7or8bit)
except NameError:
rootpkg = MIMENonMultipart("application", "xop+xml")
rootpkg.set_payload(envelope)
encode_7or8bit(rootpkg)
# Set up multipart headers.
del(mtompkg['mime-version'])
mtompkg.set_param('start-info', roottype)
mtompkg.set_param('start', '<soaplibEnvelope>')
if 'SOAPAction' in headers:
mtompkg.add_header('SOAPAction', headers.get('SOAPAction'))
# Set up root SOAP part headers.
del(rootpkg['mime-version'])
rootpkg.add_header('Content-ID', '<soaplibEnvelope>')
for n, v in rootparams.items():
rootpkg.set_param(n, v)
rootpkg.set_param('type', roottype)
mtompkg.attach(rootpkg)
# Extract attachments from SOAP envelope.
for i in range(len(params)):
name, typ = params[i]
if typ == Attachment:
id = "soaplibAttachment_%s" % (len(mtompkg.get_payload()), )
param = message[i]
param.text = ""
incl = create_xml_subelement(param,
"{http://www.w3.org/2004/08/xop/include}Include")
incl.attrib["href"] = "cid:%s" % id
if paramvals[i].fileName and not paramvals[i].data:
paramvals[i].load_from_file()
data = paramvals[i].data
attachment = None
try:
attachment = MIMEApplication(data, _encoder=encode_7or8bit)
except NameError:
attachment = MIMENonMultipart("application", "octet-stream")
attachment.set_payload(data)
encode_7or8bit(attachment)
del(attachment['mime-version'])
attachment.add_header('Content-ID', '<%s>' % (id, ))
mtompkg.attach(attachment)
# Update SOAP envelope.
soapmsg.close()
soapmsg = StringIO()
soaptree.write(soapmsg)
rootpkg.set_payload(soapmsg.getvalue())
soapmsg.close()
# extract body string from MIMEMultipart message
bound = '--%s' % (mtompkg.get_boundary(), )
marray = mtompkg.as_string().split(bound)
mtombody = bound
mtombody += bound.join(marray[1:])
# set Content-Length
mtompkg.add_header("Content-Length", str(len(mtombody)))
# extract dictionary of headers from MIMEMultipart message
mtomheaders = {}
for name, value in mtompkg.items():
mtomheaders[name] = value
if len(mtompkg.get_payload()) <= 1:
return (headers, envelope)
return (mtomheaders, mtombody)
def run(self):
if len(self.args) == 0:
raise UsageError("needs 1 argument")
self.server = None
self.dataDir = "."
self.count_ok = 0
self.count_nok = 0
if self.options.host is None:
raise UsageError("--host must be specified")
## for fn in self.attach_files:
## if not os.path.exists(fn):
## raise OperationFailed("File %s does not exist."%fn)
files = []
for pattern in self.args:
files += glob.glob(pattern)
self.count_todo = len(files)
if self.count_todo == 0:
self.notice("Nothing to do: no input files found.")
return
if self.options.recipient is None:
recipients = []
else:
recipients = getaddresses([self.options.recipient])
sender = self.options.sender
subject = self.options.subject
"""
if the first input file's name ends with .eml, then this
becomes the outer message
"""
if files[0].lower().endswith(".eml"):
self.notice("Reading file %s...", files[0])
first = email.message_from_file(codecs.open(files[0], "r", self.options.encoding), _class=MyMessage)
if first.has_key("subject"):
subject = first["subject"]
del first["subject"]
first["subject"] = Header(subject, first.get_charset())
self.encodeaddrs(first, "from")
if sender is None:
sender = first["from"]
self.encodeaddrs(first, "to", recipients)
self.encodeaddrs(first, "cc", recipients)
self.encodeaddrs(first, "bcc", recipients)
del first["bcc"]
del files[0]
self.count_todo -= 1
else:
first = None
if len(files) == 0:
outer = first
else:
# Create the enclosing (outer) message
outer = MIMEMultipart()
# outer.preamble = 'You will not see this in a MIME-aware mail reader.\n'
if first is not None:
first.add_header("Content-Disposition", "inline")
outer.attach(first)
for hdr in ("to", "cc"):
outer[hdr] = first[hdr]
outer["subject"] = subject
outer["from"] = sender
self.notice("Attaching %d files...", self.count_todo)
i = 1
for filename in files:
self.notice(u"%s (%d/%d)", filename, i, self.count_todo)
part = self.file2msg(filename)
# Set the filename parameter
part.add_header("Content-Disposition", "attachment", filename=os.path.basename(filename))
outer.attach(part)
i += 1
# for part in outer.walk():
# if recipient is None: recipient=part["To"]
# if bcc is None: bcc=part["Bcc"]
if self.options.subject is not None:
del outer["subject"]
outer["subject"] = self.options.subject
# if self.options.sender is not None:
# del outer['from']
# outer['from'] = self.options.sender
# del outer['to']
# outer['to'] = recipient
# if bcc is not None:
# outer['Bcc'] = bcc
# print "Bcc:", bcc
# headers_i18n(outer)
if len(recipients) == 0:
for addr in open(opj(self.dataDir, "addrlist.txt")).xreadlines():
addr = addr.strip()
if len(addr) != 0 and addr[0] != "#":
recipients += getaddresses([addr])
if not outer.has_key("Subject"):
raise "Subject header is missing"
if not outer.has_key("Date"):
outer["Date"] = email.Utils.formatdate(None, True)
for k, v in outer.items():
print k, ":", unicode(v)
# self.notice(str(outer.keys()))
self.notice(_("Message size: %d bytes."), len(str(outer)))
self.notice(_("Send this to %d recipients: %s"), len(recipients), ", ".join([a[1] for a in recipients]))
sender = parseaddr(unicode(sender))[1]
# print "sender:", unicode(sender)
# print outer.as_string(unixfrom=0)
if not self.confirm("Okay?"):
return
self.connect()
self.sendmsg(outer, sender, recipients)
self.server.quit()
self.notice(_("Sent to %d recipients."), self.count_ok)
if self.count_nok != 0:
self.notice(_("%d recipients refused."), self.count_nok)
def make_multipart(content, default_encoding="ascii", with_filenames=False):
"""Return the headers and content for multipart/form-data.
content -- Dict with content to POST. The dict values are expected to
be unicode or decodable with us-ascii.
default_encoding -- Send multipart with this encoding, if no special
encoding was given with the content. Default is ascii.
with_filenames -- If True, a multipart's files will be sent with the
filename paramenter set. Default is False.
"""
def add_disposition(part, name, filename=None, disposition="form-data"):
"""Add a Content-Disposition header to the part.
part -- Part to add header to.
name -- Name of the part.
filename -- Add this filename as parameter, if given.
disposition -- Value of the content-disposition header.
"""
# RFC 2388 Returning Values from Forms: multipart/form-data
# Each part is expected to contain a content-disposition header
# [RFC 2183] where the disposition type is "form-data", and where the
# disposition contains an (additional) parameter of "name", where the
# value of that parameter is the original field name in the form.
params = dict(name=name)
if with_filenames and (filename is not None):
# RFC 2388 Returning Values from Forms: multipart/form-data
# The original local file name may be supplied as well, either as
# a "filename" parameter either of the "content-disposition:
# form-data" header or, in the case of multiple files, in a
# "content-disposition: file" header of the subpart.
params["filename"] = path.basename(filename)
part.add_header("Content-Disposition", disposition, **params)
def create_part(key, fileobject, content_type, multiple=False):
"""Create and return a multipart part as to given file data.
key -- Field name.
fileobject -- The file-like object to add to the part.
content_type -- Content-type of the file. If None, use default.
multiple -- If true, use Content-Disposition: file.
"""
if not content_type:
content_type = DEFAULT_CONTENT_TYPE
(maintype, subtype) = content_type.split("/")
part = MIMEBase(maintype, subtype)
part.set_payload(fileobject.read())
encode_base64(part)
filename = getattr(fileobject, "name", None)
kwargs = dict()
if multiple:
# RFC 2388 Returning Values from Forms: multipart/form-data
# The original local file name may be supplied as well, either as
# a "filename" parameter either of the "content-disposition:
# form-data" header or, in the case of multiple files, in a
# "content-disposition: file" header of the subpart.
kwargs["disposition"] = "file"
add_disposition(part, key, filename, **kwargs)
return part
# RFC 2388 Returning Values from Forms: multipart/form-data
mime = MIMEMultipart("form-data")
files = list()
items_iterator = content.iteritems() if PYTHON2 else content.items()
for (key, data) in items_iterator:
# Are there explicit encodings/content-types given?
# Note: Cannot do a (value, encoding) = value here as fileobjects then
# would get iterated, which is not what we want.
if isinstance(data, tuple) and (len(data) == 2):
(value, encoding) = data
else:
(value, encoding) = (data, None)
# collect file-like objects
if hasattr(value, "read"):
files.append((key, value, encoding))
# no file-like object
else:
if isinstance(value, MIMEBase):
part = value
else:
encoding = encoding if encoding else default_encoding
part = MIMEText(value, "plain", encoding)
add_disposition(part, key)
mime.attach(part)
filecount = len(files)
if filecount == 1:
filedata = files[0]
part = create_part(*filedata)
mime.attach(part)
elif filecount > 1:
# RFC 2388 Returning Values from Forms: multipart/form-data
# 4.2 Sets of files
# If the value of a form field is a set of files rather than a single
# file, that value can be transferred together using the
# "multipart/mixed" format.
mixed = MIMEMultipart("mixed")
for filedata in files:
part = create_part(multiple=True, *filedata)
mixed.attach(part)
mime.attach(mixed)
# mime.items must be called after mime.as_string when the headers shall
# contain the boundary
complete_mime = mime.as_string()
headers = dict(mime.items())
# trim headers from create mime as these will later be added by httplib.
payload_start = complete_mime.index("\n\n") + 2
payload = complete_mime[payload_start:]
return (headers, payload)
def gdrive_upload(filename, mime_type, fileobj):
"""
Function: gdrive_upload
Summary: Uploads given fileobj of mime_type
to Google Drive, titling it filename
Examples:
gd_resp = gdrive_upload(file_name,
file_mime,
gen_participants_xlsx(part_data).getvalue())
Attributes:
@param (filename):str
@param (mime_type):str
@param (fileobj):bytes-like object
Returns: JSON response from Google Drive API
"""
# Alternative way:
# https://developers.google.com/drive/v2/web/savetodrive
# Btw, there's another alternative way, this button downloads file
# to the user's browser and uploads it to Google Drive then:
# https://developers.google.com/drive/v2/web/savetodrive
try:
msg = MIMEMultipart('related')
# Add file metadata
msg.attach(MIMEApplication(
json_dumps({
'title': filename,
'mimeType': mime_type}),
'json',
_encoder=encode_noop))
# Add file itself
msg.attach(MIMEApplication(
fileobj,
mime_type))
google_api = pub('google-api')
# Send file to drive with exponential backoff enabled:
# https://developers.google.com/drive/v2/web/manage-uploads#exp-backoff
retries_count = 0
while True:
try:
gd_rsrc = google_api.post(
'https://www.googleapis.com/upload/drive/v2/files'
'?uploadType=multipart&convert=true',
data=msg.as_string(),
headers=dict(msg.items()))
break
except requests.exceptions.HTTPError as http_error:
if gd_rsrc.status < 500 or retries_count > 3:
raise http_error
# Wait exponential time and then retry
time.sleep(2 ** retries_count + random.random())
retries_count += 1
return gd_rsrc.json()
except Exception as e:
raise HTTPError(500) from e
logger.error('Error sending to drive: {} {} ({})'.format(
gd_rsrc.status, gd_rsrc.reason, msg))
raise HTTPError(500,
{'message': gd_rsrc.status, 'data': msg}) from e
class Mdn:
"""Class for handling AS2 MDNs. Includes functions for both
parsing and building them.
"""
def __init__(self, mdn_mode=None, digest_alg=None, mdn_url=None):
self.message_id = None
self.orig_message_id = None
self.payload = None
self.mdn_mode = mdn_mode
self.digest_alg = digest_alg
self.mdn_url = mdn_url
@property
def content(self):
"""Function returns the body of the mdn message as a byte string"""
if self.payload is not None:
message_bytes = mime_to_bytes(self.payload)
boundary = b"--" + self.payload.get_boundary().encode("utf-8")
temp = message_bytes.split(boundary)
temp.pop(0)
return boundary + boundary.join(temp)
return ""
@property
def headers(self):
"""Return the headers in the payload as a dictionary."""
if self.payload:
return dict(self.payload.items())
return {}
@property
def headers_str(self):
"""Return the headers in the payload as a string."""
message_header = ""
if self.payload:
for k, v in self.headers.items():
message_header += f"{k}: {v}\r\n"
return message_header.encode("utf-8")
def build(
self,
message,
status,
detailed_status=None,
confirmation_text=MDN_CONFIRM_TEXT,
failed_text=MDN_FAILED_TEXT,
):
"""Function builds and signs an AS2 MDN message.
:param message: The received AS2 message for which this is an MDN.
:param status: The status of processing of the received AS2 message.
:param detailed_status: The optional detailed status of processing of the received AS2
message. Used to give additional error info (default "None")
:param confirmation_text: The confirmation message sent in the first part of the MDN.
:param failed_text: The failure message sent in the first part of the failed MDN.
"""
# Generate message id using UUID 1 as it uses both hostname and time
self.message_id = email_utils.make_msgid().lstrip("<").rstrip(">")
self.orig_message_id = message.message_id
# Set up the message headers
mdn_headers = {
"AS2-Version": AS2_VERSION,
"ediint-features": EDIINT_FEATURES,
"Message-ID": f"<{self.message_id}>",
"AS2-From": quote_as2name(message.headers.get("as2-to")),
"AS2-To": quote_as2name(message.headers.get("as2-from")),
"Date": email_utils.formatdate(localtime=True),
"user-agent": "pyAS2 Open Source AS2 Software",
}
# Set the confirmation text message here
# overwrite with organization specific message
if message.receiver and message.receiver.mdn_confirm_text:
confirmation_text = message.receiver.mdn_confirm_text
# overwrite with partner specific message
if message.sender and message.sender.mdn_confirm_text:
confirmation_text = message.sender.mdn_confirm_text
if status != "processed":
confirmation_text = failed_text
self.payload = MIMEMultipart("report",
report_type="disposition-notification")
# Create and attach the MDN Text Message
mdn_text = email_message.Message()
mdn_text.set_payload(f"{confirmation_text}\r\n")
mdn_text.set_type("text/plain")
del mdn_text["MIME-Version"]
encoders.encode_7or8bit(mdn_text)
self.payload.attach(mdn_text)
# Create and attache the MDN Report Message
mdn_base = email_message.Message()
mdn_base.set_type("message/disposition-notification")
mdn_report = "Reporting-UA: pyAS2 Open Source AS2 Software\r\n"
mdn_report += f'Original-Recipient: rfc822; {message.headers.get("as2-to")}\r\n'
mdn_report += f'Final-Recipient: rfc822; {message.headers.get("as2-to")}\r\n'
mdn_report += f"Original-Message-ID: <{message.message_id}>\r\n"
mdn_report += f"Disposition: automatic-action/MDN-sent-automatically; {status}"
if detailed_status:
mdn_report += f": {detailed_status}"
mdn_report += "\r\n"
if message.mic:
mdn_report += f"Received-content-MIC: {message.mic.decode()}, {message.digest_alg}\r\n"
mdn_base.set_payload(mdn_report)
del mdn_base["MIME-Version"]
encoders.encode_7or8bit(mdn_base)
self.payload.attach(mdn_base)
logger.debug(
f"MDN report for message {message.message_id} created:\n{mime_to_bytes(mdn_base)}"
)
# Sign the MDN if it is requested by the sender
if (message.headers.get("disposition-notification-options")
and message.receiver and message.receiver.sign_key):
self.digest_alg = (
message.headers["disposition-notification-options"].split(
";")[-1].split(",")[-1].strip().replace("-", ""))
signed_mdn = MIMEMultipart("signed",
protocol="application/pkcs7-signature")
del signed_mdn["MIME-Version"]
signed_mdn.attach(self.payload)
# Create the signature mime message
signature = email_message.Message()
signature.set_type("application/pkcs7-signature")
signature.set_param("name", "smime.p7s")
signature.set_param("smime-type", "signed-data")
signature.add_header("Content-Disposition",
"attachment",
filename="smime.p7s")
del signature["MIME-Version"]
signed_data = sign_message(canonicalize(self.payload),
self.digest_alg,
message.receiver.sign_key)
signature.set_payload(signed_data)
encoders.encode_base64(signature)
signed_mdn.set_param("micalg", self.digest_alg)
signed_mdn.attach(signature)
self.payload = signed_mdn
logger.debug(f"Signing the MDN for message {message.message_id}")
# Update the headers of the final payload and set message boundary
for k, v in mdn_headers.items():
if self.payload.get(k):
self.payload.replace_header(k, v)
else:
self.payload.add_header(k, v)
self.payload.set_boundary(make_mime_boundary())
logger.debug(f"MDN generated for message {message.message_id} with "
f"content:\n {mime_to_bytes(self.payload)}")
def parse(self, raw_content, find_message_cb):
"""Function parses the RAW AS2 MDN, verifies it and extracts the
processing status of the orginal AS2 message.
:param raw_content:
A byte string of the received HTTP headers followed by the body.
:param find_message_cb:
A callback the must returns the original Message Object. The
original message-id and original recipient AS2 ID are passed
as arguments to it.
:returns:
A two element tuple containing (status, detailed_status). The
status is a string indicating the status of the transaction. The
optional detailed_status gives additional information about the
processing status.
"""
status, detailed_status = None, None
try:
self.payload = parse_mime(raw_content)
self.orig_message_id, orig_recipient = self.detect_mdn()
# Call the find message callback which should return a Message instance
orig_message = find_message_cb(self.orig_message_id,
orig_recipient)
# Extract the headers and save it
mdn_headers = {}
for k, v in self.payload.items():
k = k.lower()
if k == "message-id":
self.message_id = v.lstrip("<").rstrip(">")
mdn_headers[k] = v
if (orig_message.receiver.mdn_digest_alg
and self.payload.get_content_type() != "multipart/signed"):
status = "failed/Failure"
detailed_status = "Expected signed MDN but unsigned MDN returned"
return status, detailed_status
if self.payload.get_content_type() == "multipart/signed":
logger.debug(
f"Verifying signed MDN: \n{mime_to_bytes(self.payload)}")
message_boundary = (
"--" + self.payload.get_boundary()).encode("utf-8")
# Extract the signature and the signed payload
signature = None
signature_types = [
"application/pkcs7-signature",
"application/x-pkcs7-signature",
]
for part in self.payload.walk():
if part.get_content_type() in signature_types:
signature = part.get_payload(decode=True)
elif part.get_content_type() == "multipart/report":
self.payload = part
# Verify the message, first using raw message and if it fails
# then convert to canonical form and try again
mic_content = extract_first_part(raw_content, message_boundary)
verify_cert = orig_message.receiver.load_verify_cert()
try:
self.digest_alg = verify_message(mic_content, signature,
verify_cert)
except IntegrityError:
mic_content = canonicalize(self.payload)
self.digest_alg = verify_message(mic_content, signature,
verify_cert)
for part in self.payload.walk():
if part.get_content_type(
) == "message/disposition-notification":
logger.debug(
f"MDN report for message {orig_message.message_id}:\n{part.as_string()}"
)
mdn = part.get_payload()[-1]
mdn_status = mdn["Disposition"].split(
";").pop().strip().split(":")
status = mdn_status[0]
if status == "processed":
# Compare the original mic with the received mic
mdn_mic = mdn.get("Received-Content-MIC",
"").split(",")[0]
if (mdn_mic and orig_message.mic
and mdn_mic != orig_message.mic.decode()):
status = "processed/warning"
detailed_status = "Message Integrity check failed."
else:
detailed_status = " ".join(mdn_status[1:]).strip()
except MDNNotFound:
status = "failed/Failure"
detailed_status = "mdn-not-found"
except Exception as e: # pylint: disable=W0703
status = "failed/Failure"
detailed_status = f"Failed to parse received MDN. {e}"
logger.error(
f"Failed to parse AS2 MDN\n: {traceback.format_exc()}")
return status, detailed_status
def detect_mdn(self):
"""Function checks if the received raw message is an AS2 MDN or not.
:raises MDNNotFound: If the received payload is not an MDN then this
exception is raised.
:return:
A two element tuple containing (message_id, message_recipient). The
message_id is the original AS2 message id and the message_recipient
is the original AS2 message recipient.
"""
mdn_message = None
if self.payload.get_content_type() == "multipart/report":
mdn_message = self.payload
elif self.payload.get_content_type() == "multipart/signed":
for part in self.payload.walk():
if part.get_content_type() == "multipart/report":
mdn_message = self.payload
if not mdn_message:
raise MDNNotFound("No MDN found in the received message")
message_id, message_recipient = None, None
for part in mdn_message.walk():
if part.get_content_type() == "message/disposition-notification":
mdn = part.get_payload()[0]
message_id = mdn.get("Original-Message-ID").strip("<>")
message_recipient = None
if "Original-Recipient" in mdn:
message_recipient = mdn["Original-Recipient"].split(
";")[1].strip()
elif "Final-Recipient" in mdn:
message_recipient = mdn["Final-Recipient"].split(
";")[1].strip()
return message_id, message_recipient
def make_multipart(content, default_encoding="ascii", with_filenames=False):
"""Return the headers and content for multipart/form-data.
content -- Dict with content to POST. The dict values are expected to
be unicode or decodable with us-ascii.
default_encoding -- Send multipart with this encoding, if no special
encoding was given with the content. Default is ascii.
with_filenames -- If True, a multipart's files will be sent with the
filename paramenter set. Default is False.
"""
def add_disposition(part, name, filename=None, disposition="form-data"):
"""Add a Content-Disposition header to the part.
part -- Part to add header to.
name -- Name of the part.
filename -- Add this filename as parameter, if given.
disposition -- Value of the content-disposition header.
"""
# RFC 2388 Returning Values from Forms: multipart/form-data
# Each part is expected to contain a content-disposition header
# [RFC 2183] where the disposition type is "form-data", and where the
# disposition contains an (additional) parameter of "name", where the
# value of that parameter is the original field name in the form.
params = dict(name=name)
if with_filenames and (filename is not None):
# RFC 2388 Returning Values from Forms: multipart/form-data
# The original local file name may be supplied as well, either as
# a "filename" parameter either of the "content-disposition:
# form-data" header or, in the case of multiple files, in a
# "content-disposition: file" header of the subpart.
params["filename"] = path.basename(filename)
part.add_header("Content-Disposition", disposition, **params)
def create_part(key, fileobject, content_type, multiple=False):
"""Create and return a multipart part as to given file data.
key -- Field name.
fileobject -- The file-like object to add to the part.
content_type -- Content-type of the file. If None, use default.
multiple -- If true, use Content-Disposition: file.
"""
if not content_type:
content_type = DEFAULT_CONTENT_TYPE
(maintype, subtype) = content_type.split("/")
part = MIMEBase(maintype, subtype)
part.set_payload(fileobject.read())
encode_base64(part)
filename = getattr(fileobject, "name", None)
kwargs = dict()
if multiple:
# RFC 2388 Returning Values from Forms: multipart/form-data
# The original local file name may be supplied as well, either as
# a "filename" parameter either of the "content-disposition:
# form-data" header or, in the case of multiple files, in a
# "content-disposition: file" header of the subpart.
kwargs["disposition"] = "file"
add_disposition(part, key, filename, **kwargs)
return part
# RFC 2388 Returning Values from Forms: multipart/form-data
mime = MIMEMultipart("form-data")
files = list()
items_iterator = content.iteritems() if PYTHON2 else content.items()
for (key, data) in items_iterator:
# Are there explicit encodings/content-types given?
# Note: Cannot do a (value, encoding) = value here as fileobjects then
# would get iterated, which is not what we want.
if isinstance(data, tuple) and (len(data) == 2):
(value, encoding) = data
else:
(value, encoding) = (data, None)
# collect file-like objects
if hasattr(value, "read"):
files.append((key, value, encoding))
# no file-like object
else:
if isinstance(value, MIMEBase):
part = value
else:
encoding = encoding if encoding else default_encoding
part = MIMEText(value, "plain", encoding)
add_disposition(part, key)
mime.attach(part)
filecount = len(files)
if filecount == 1:
filedata = files[0]
part = create_part(*filedata)
mime.attach(part)
elif filecount > 1:
# RFC 2388 Returning Values from Forms: multipart/form-data
# 4.2 Sets of files
# If the value of a form field is a set of files rather than a single
# file, that value can be transferred together using the
# "multipart/mixed" format.
mixed = MIMEMultipart("mixed")
for filedata in files:
part = create_part(multiple=True, *filedata)
mixed.attach(part)
mime.attach(mixed)
# mime.items must be called after mime.as_string when the headers shall
# contain the boundary
complete_mime = mime.as_string()
headers = dict(mime.items())
# trim headers from create mime as these will later be added by httplib.
payload_start = complete_mime.index("\n\n") + 2
payload = complete_mime[payload_start:]
return (headers, payload)
def submit(self):
"""Submit a query and parse the response.
@return: The data retrieved from api.php (a dict)
"""
while True:
paramstring = self.http_params()
action = self.params.get("action", "")
simulate = self._simulate(action)
if simulate:
return simulate
self.site.throttle(write=self.write)
uri = self.site.scriptpath() + "/api.php"
ssl = False
if self.site.family.name in config.available_ssl_project:
if action == "login" and config.use_SSL_onlogin:
ssl = True
elif config.use_SSL_always:
ssl = True
try:
if self.mime:
# construct a MIME message containing all API key/values
container = MIMEMultipart(_subtype='form-data')
for key in self.params:
# key "file" requires special treatment in a multipart
# message
if key == "file":
local_filename = self.params[key]
filetype = mimetypes.guess_type(local_filename)[0] \
or 'application/octet-stream'
file_content = file(local_filename, "rb").read()
submsg = MIMENonMultipart(*filetype.split("/"))
submsg.add_header("Content-disposition",
"form-data",
name=key,
filename=local_filename)
submsg.set_payload(file_content)
else:
try:
self.params[key].encode("ascii")
keytype = ("text", "plain")
except UnicodeError:
keytype = ("application", "octet-stream")
submsg = MIMENonMultipart(*keytype)
submsg.add_header("Content-disposition",
"form-data",
name=key)
submsg.set_payload(self.params[key])
container.attach(submsg)
# strip the headers to get the HTTP message body
body = container.as_string()
marker = "\n\n" # separates headers from body
eoh = body.find(marker)
body = body[eoh + len(marker):]
# retrieve the headers from the MIME object
mimehead = dict(list(container.items()))
rawdata = http.request(self.site,
uri,
ssl,
method="POST",
headers=mimehead,
body=body)
else:
rawdata = http.request(
self.site,
uri,
ssl,
method="POST",
headers={
'Content-Type': 'application/x-www-form-urlencoded'
},
body=paramstring)
# import traceback
# traceback.print_stack()
# print rawdata
except Server504Error:
pywikibot.log(u"Caught HTTP 504 error; retrying")
self.wait()
continue
except FatalServerError:
# This error is not going to be fixed by just waiting
pywikibot.error(traceback.format_exc())
raise
# TODO: what other exceptions can occur here?
except Exception:
# for any other error on the http request, wait and retry
pywikibot.error(traceback.format_exc())
pywikibot.log(u"%s, %s" % (uri, paramstring))
self.wait()
continue
if not isinstance(rawdata, unicode):
rawdata = rawdata.decode(self.site.encoding())
pywikibot.debug(u"API response received:\n" + rawdata, _logger)
if rawdata.startswith(u"unknown_action"):
raise APIError(rawdata[:14], rawdata[16:])
try:
result = json.loads(rawdata)
except ValueError:
# if the result isn't valid JSON, there must be a server
# problem. Wait a few seconds and try again
pywikibot.warning(
"Non-JSON response received from server %s; the server may be down."
% self.site)
pywikibot.debug(rawdata, _logger)
# there might also be an overflow, so try a smaller limit
for param in self.params:
if param.endswith("limit"):
value = self.params[param]
try:
self.params[param] = str(int(value) // 2)
pywikibot.output(u"Set %s = %s" %
(param, self.params[param]))
except:
pass
self.wait()
continue
if not result:
result = {}
if not isinstance(result, dict):
raise APIError("Unknown",
"Unable to process query response of type %s." %
type(result),
data=result)
if self['action'] == 'query':
if 'userinfo' in result.get('query', ()):
if hasattr(self.site, '_userinfo'):
self.site._userinfo.update(result['query']['userinfo'])
else:
self.site._userinfo = result['query']['userinfo']
status = self.site._loginstatus # save previous login status
if (("error" in result
and result["error"]["code"].endswith("limit"))
or (status >= 0 and self.site._userinfo['name'] !=
self.site._username[status])):
# user is no longer logged in (session expired?)
# reset userinfo, then make user log in again
del self.site._userinfo
self.site._loginstatus = -1
if status < 0:
status = 0 # default to non-sysop login
self.site.login(status)
# retry the previous query
continue
if "warnings" in result:
modules = [k for k in result["warnings"] if k != "info"]
for mod in modules:
if '*' in result["warnings"][mod]:
text = result["warnings"][mod]['*']
elif 'html' in result["warnings"][mod]:
# Bugzilla 49978
text = result["warnings"][mod]['html']['*']
else:
# This is just a warning, we shouldn't raise an
# exception because of it
continue
pywikibot.warning(u"API warning (%s): %s" % (mod, text))
if "error" not in result:
return result
if "*" in result["error"]:
# help text returned
result['error']['help'] = result['error'].pop("*")
code = result["error"].pop("code", "Unknown")
info = result["error"].pop("info", None)
if code == "maxlag":
lag = lagpattern.search(info)
if lag:
pywikibot.log(u"Pausing due to database lag: " + info)
self.site.throttle.lag(int(lag.group("lag")))
continue
if code.startswith(u'internal_api_error_'):
self.wait()
continue
# bugs 46535, 62126, 64494
# maybe removed when it 46535 is solved
if code == "failed-save" and action == 'wbeditentity':
try:
message = result["error"]["messages"]["0"]["name"]
except KeyError:
message = None
if message == u'edit-already-exists':
self.wait()
continue
# raise error
try:
pywikibot.log(u"API Error: query=\n%s" %
pprint.pformat(self.params))
pywikibot.log(u" response=\n%s" % result)
raise APIError(code, info, **result["error"])
except TypeError:
raise RuntimeError(result)
def run(self):
if len(self.args) == 0:
raise UsageError("needs 1 argument")
self.server = None
self.dataDir = '.'
self.count_ok = 0
self.count_nok = 0
if self.options.host is None:
raise UsageError("--host must be specified")
## for fn in self.attach_files:
## if not os.path.exists(fn):
## raise OperationFailed("File %s does not exist."%fn)
files = []
for pattern in self.args:
files += glob.glob(pattern)
self.count_todo = len(files)
if self.count_todo == 0:
self.notice("Nothing to do: no input files found.")
return
if self.options.recipient is None:
recipients = []
else:
recipients = getaddresses([self.options.recipient])
sender = self.options.sender
subject = self.options.subject
"""
if the first input file's name ends with .eml, then this
becomes the outer message
"""
if files[0].lower().endswith(".eml"):
self.notice("Reading file %s...", files[0])
first = email.message_from_file(codecs.open(
files[0], "r", self.options.encoding),
_class=MyMessage)
if first.has_key('subject'):
subject = first["subject"]
del first["subject"]
first["subject"] = Header(subject, first.get_charset())
self.encodeaddrs(first, 'from')
if sender is None: sender = first["from"]
self.encodeaddrs(first, 'to', recipients)
self.encodeaddrs(first, 'cc', recipients)
self.encodeaddrs(first, 'bcc', recipients)
del first['bcc']
del files[0]
self.count_todo -= 1
else:
first = None
if len(files) == 0:
outer = first
else:
# Create the enclosing (outer) message
outer = MIMEMultipart()
# outer.preamble = 'You will not see this in a MIME-aware mail reader.\n'
if first is not None:
first.add_header('Content-Disposition', 'inline')
outer.attach(first)
for hdr in ('to', 'cc'):
outer[hdr] = first[hdr]
outer['subject'] = subject
outer['from'] = sender
self.notice("Attaching %d files...", self.count_todo)
i = 1
for filename in files:
self.notice(u"%s (%d/%d)", filename, i, self.count_todo)
part = self.file2msg(filename)
# Set the filename parameter
part.add_header('Content-Disposition',
'attachment',
filename=os.path.basename(filename))
outer.attach(part)
i += 1
#for part in outer.walk():
#if recipient is None: recipient=part["To"]
#if bcc is None: bcc=part["Bcc"]
if self.options.subject is not None:
del outer['subject']
outer['subject'] = self.options.subject
#if self.options.sender is not None:
# del outer['from']
# outer['from'] = self.options.sender
#del outer['to']
#outer['to'] = recipient
#if bcc is not None:
# outer['Bcc'] = bcc
# print "Bcc:", bcc
#headers_i18n(outer)
if len(recipients) == 0:
for addr in open(opj(self.dataDir, "addrlist.txt")).xreadlines():
addr = addr.strip()
if len(addr) != 0 and addr[0] != "#":
recipients += getaddresses([addr])
if not outer.has_key("Subject"):
raise "Subject header is missing"
if not outer.has_key("Date"):
outer["Date"] = email.Utils.formatdate(None, True)
for k, v in outer.items():
print k, ":", unicode(v)
#self.notice(str(outer.keys()))
self.notice(_("Message size: %d bytes."), len(str(outer)))
self.notice(_("Send this to %d recipients: %s"), len(recipients),
", ".join([a[1] for a in recipients]))
sender = parseaddr(unicode(sender))[1]
# print "sender:", unicode(sender)
# print outer.as_string(unixfrom=0)
if not self.confirm("Okay?"):
return
self.connect()
self.sendmsg(outer, sender, recipients)
self.server.quit()
self.notice(_("Sent to %d recipients."), self.count_ok)
if self.count_nok != 0:
self.notice(_("%d recipients refused."), self.count_nok)
class Mail(object):
def __init__(self,
window,
parent,
subject=None,
to=None,
_from=None,
msg=None):
self.window = window
self.parent = parent
self.msg = MIMEMultipart()
if not subject == None:
self.msg['Subject'] = subject
if not to == None:
self.msg['To'] = to
if not _from == None:
self.msg['From'] = _from
if not msg == None:
self.body = MIMEText(msg, 'html')
self.msg.attach(self.body)
def attach_file(self, filenames):
self.filenames = filenames
for i in self.filenames:
file = MIMEApplication(open(i, 'rb').read())
file.add_header('Content-Disposition',
'attachment',
filename=i.split('/')[-1])
self.msg.attach(file)
def send_mail(self):
self.msg['Subject'] = self.parent.e_subject.get()
self.msg['To'] = self.parent.e_to.get()
self.msg[
'From'] = self.parent.this_address + 'no address' #remove for production
self.msg['CC'] = self.parent.e_cc.get()
self.body = MIMEText(self.parent.t_body.get(1.0, tk.END), 'html')
self.msg.attach(self.body)
print(self.msg.items())
"""
try:
self.connect()
#send mail here
except:
pass
#shit happens
finally:
try:
self.server.quit()
except:
#whatever idc
pass
"""
def connect(self):
self.server = smtplib.SMTP('smtp.office365.com', 587)
self.server.ehlo()
self.server.starttls(context=ssl.create_default_context())
self.server.login(
keyring.get_password(self.parent.service_name, USER_RECOVER_KEY),
keyring.get_password(
self.parent.service_name,
keyring.get_password(self.parent.service_name,
USER_RECOVER_KEY)))
def submit(self):
"""Submit a query and parse the response.
@return: a dict containing data retrieved from api.php
"""
while True:
paramstring = self.http_params()
action = self.params.get("action", "")
simulate = self._simulate(action)
if simulate:
return simulate
if self.throttle:
self.site.throttle(write=self.write)
else:
pywikibot.log("Action '{0}' is submitted not throttled.".format(action))
uri = self.site.scriptpath() + "/api.php"
try:
if self.mime:
# construct a MIME message containing all API key/values
container = MIMEMultipart(_subtype='form-data')
for key in self.params:
# key "file" requires special treatment in a multipart
# message
if key == "file":
local_filename = self.params[key]
filetype = mimetypes.guess_type(local_filename)[0] \
or 'application/octet-stream'
file_content = file(local_filename, "rb").read()
submsg = Request._generate_MIME_part(
key, file_content, filetype.split('/'),
{'filename': local_filename})
else:
submsg = Request._generate_MIME_part(
key, self.params[key], None, None)
container.attach(submsg)
for key, value in self.mime_params.items():
container.attach(Request._generate_MIME_part(key, *value))
# strip the headers to get the HTTP message body
body = container.as_string()
marker = "\n\n" # separates headers from body
eoh = body.find(marker)
body = body[eoh + len(marker):]
# retrieve the headers from the MIME object
headers = dict(list(container.items()))
else:
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
body = paramstring
rawdata = http.request(
self.site, uri, method="POST",
headers=headers, body=body)
# import traceback
# traceback.print_stack()
# print rawdata
except Server504Error:
pywikibot.log(u"Caught HTTP 504 error; retrying")
self.wait()
continue
except FatalServerError:
# This error is not going to be fixed by just waiting
pywikibot.error(traceback.format_exc())
raise
# TODO: what other exceptions can occur here?
except Exception:
# for any other error on the http request, wait and retry
pywikibot.error(traceback.format_exc())
pywikibot.log(u"%s, %s" % (uri, paramstring))
self.wait()
continue
if not isinstance(rawdata, unicode):
rawdata = rawdata.decode(self.site.encoding())
pywikibot.debug(u"API response received:\n" + rawdata, _logger)
if rawdata.startswith(u"unknown_action"):
raise APIError(rawdata[:14], rawdata[16:])
try:
result = json.loads(rawdata)
except ValueError:
# if the result isn't valid JSON, there must be a server
# problem. Wait a few seconds and try again
pywikibot.warning(
"Non-JSON response received from server %s; the server may be down."
% self.site)
pywikibot.debug(rawdata, _logger)
# there might also be an overflow, so try a smaller limit
for param in self.params:
if param.endswith("limit"):
value = self.params[param]
try:
self.params[param] = str(int(value) // 2)
pywikibot.output(u"Set %s = %s"
% (param, self.params[param]))
except:
pass
self.wait()
continue
if not result:
result = {}
if not isinstance(result, dict):
raise APIError("Unknown",
"Unable to process query response of type %s."
% type(result),
data=result)
if self['action'] == 'query':
if 'userinfo' in result.get('query', ()):
if hasattr(self.site, '_userinfo'):
self.site._userinfo.update(result['query']['userinfo'])
else:
self.site._userinfo = result['query']['userinfo']
status = self.site._loginstatus # save previous login status
if (("error" in result
and result["error"]["code"].endswith("limit"))
or (status >= 0
and self.site._userinfo['name'] != self.site._username[status])):
# user is no longer logged in (session expired?)
# reset userinfo, then make user log in again
del self.site._userinfo
self.site._loginstatus = -1
if status < 0:
status = 0 # default to non-sysop login
self.site.login(status)
# retry the previous query
continue
self._handle_warnings(result)
if "error" not in result:
return result
if "*" in result["error"]:
# help text returned
result['error']['help'] = result['error'].pop("*")
code = result["error"].pop("code", "Unknown")
info = result["error"].pop("info", None)
if code == "maxlag":
lag = lagpattern.search(info)
if lag:
pywikibot.log(
u"Pausing due to database lag: " + info)
self.site.throttle.lag(int(lag.group("lag")))
continue
if code.startswith(u'internal_api_error_'):
class_name = code[len(u'internal_api_error_'):]
if class_name in ['DBConnectionError', # r 4984 & r 4580
'DBQueryError', # bug 58158
'ReadOnlyError' # bug 59227
]:
pywikibot.log(u'MediaWiki exception %s; retrying.'
% class_name)
self.wait()
continue
pywikibot.log(u"MediaWiki exception %s: query=\n%s"
% (class_name,
pprint.pformat(self.params)))
pywikibot.log(u" response=\n%s" % result)
raise APIMWException(class_name, info, **result["error"])
# bugs 46535, 62126, 64494, 66619
# maybe removed when it 46535 is solved
if code == "failed-save" and \
action == 'wbeditentity' and \
self._is_wikibase_error_retryable(result["error"]):
self.wait()
continue
# raise error
try:
pywikibot.log(u"API Error: query=\n%s"
% pprint.pformat(self.params))
pywikibot.log(u" response=\n%s"
% result)
raise APIError(code, info, **result["error"])
except TypeError:
raise RuntimeError(result)
def _build_multipart_request(self, message, xop=False):
if xop:
root = MIMEMultipart('related',
type='application/xop+xml',
start='<message>')
message_part = MIMEApplication(message,
'xop+xml',
encode_7or8bit,
type='text/xml')
else:
root = MIMEMultipart('related',
type='text/xml',
start='<message>')
message_part = MIMEBase('text', 'xml')
message_part.set_charset('UTF-8')
message_part.replace_header('Content-Transfer-Encoding', '8bit')
message_part.add_header('Content-ID', '<message>')
if xop:
message = _perform_xop_magic(message)
message_part.set_payload(message)
root.attach(message_part)
for attachment in self.active_client.attachments:
maintype, subtype = attachment['mimetype']
if maintype == 'image':
attached_part = MIMEImage(attachment['contents'],
subtype,
encode_noop,
name=attachment['filename'])
attached_part.add_header('Content-Transfer-Encoding',
'binary')
elif maintype == 'application':
attached_part = MIMEApplication(attachment['contents'],
subtype)
elif maintype == 'text':
attached_part = MIMEText(attachment['contents'],
subtype,
'utf8')
else:
attached_part = MIMEBase(maintype, subtype)
attached_part.add_header('Content-ID', '<{}>'
.format(attachment['filename']))
attached_part.add_header('Content-Disposition',
'attachment',
filename=attachment['filename'])
if attachment['http_headers']:
for header in attachment['http_headers'].keys():
_add_or_replace_http_header_if_passed(
attached_part,
attachment['http_headers'],
header)
root.attach(attached_part)
body = root.as_string().split('\n\n', 1)[1]
headers = dict(root.items())
return headers, body
def send_email(To=[r'[USERNAME]@microsoft.com'],
CC=[],
Subject='',
Body=r'',
BodyFile=r'[LogFile]',
Attachments=[],
MimeType='text/html',
From=r'[USERNAME]@microsoft.com',
Silent=True,
Strict=False,
UseService=True,
TestMode=False):
Trace(r'[To] [CC] "[Subject]" [BodyFile] [MimeType] [Attachments] UseService=[UseService] TestMode=[TestMode]')
To = to_list(To)
CC = to_list(CC)
Attachments = to_list(Attachments)
From = Expand(From)
if not Strict:
Subject = r'[ProgramName] [Action] %s ([COMPUTERNAME] [SDXROOT])' % (Subject)
Subject = Expand(Subject)
Body = Expand(Body)
BodyFile = ExpandPath(BodyFile)
To = Expand(email.utils.COMMASPACE.join(To))
CC = Expand(email.utils.COMMASPACE.join(CC))
def GetMIME(File, Prefix=''):
File = ExpandPath(File)
# Guess the content type based on the file's extension. Encoding
# will be ignored, although we should check for simple things like
# gzip'd or compressed files.
ctype, encoding = mimetypes.guess_type(File)
filename, ext = os.path.splitext(File)
if ext == r'.log':
ctype = r'text/plain'
if ctype is None or encoding is not None:
# No guess could be made, or the file is encoded (compressed), so
# use a generic bag-of-bits type.
ctype = 'application/octet-stream'
maintype, subtype = ctype.split('/', 1)
Verbose(' MimeType [ctype] for [File]')
if maintype == 'text':
fp = open(File)
# Note: we should handle calculating the charset
msg = MIMEText(Prefix + fp.read(), _subtype=subtype)
fp.close()
elif maintype == 'image':
fp = open(File, 'rb')
msg = MIMEImage(Prefix + fp.read(), _subtype=subtype)
fp.close()
elif maintype == 'audio':
fp = open(File, 'rb')
msg = MIMEAudio(Prefix + fp.read(), _subtype=subtype)
fp.close()
else:
fp = open(File, 'rb')
msg = MIMEBase(maintype, subtype)
msg.set_payload(Prefix + fp.read())
fp.close()
# Encode the payload using Base64
encoders.encode_base64(msg)
return msg
msg = MIMEMultipart()
msg['From'] = From
msg['To'] = To
msg['CC'] = CC
msg['Date'] = email.utils.formatdate(localtime=True)
msg['Subject'] = Subject
if Body and BodyFile:
maintype, subtype = MimeType.split('/', 1)
part = MIMEBase(maintype, subtype)
contents = Body
if BodyFile:
if subtype == 'html':
contents += r'<br/><br/><br/>---------------------------------------------------------<br/><br/><br/>'
else:
contents += r'\n\n\n---------------------------------------------------------\n\n\n'
file = open(BodyFile, 'r')
bodyFileContents = file.read()
if subtype == 'html':
bodyFileContents = bodyFileContents.replace('\r\n', r'<br/>')
bodyFileContents = bodyFileContents.replace('\n', r'<br/>')
contents += bodyFileContents
file.close()
part.set_payload(contents)
msg.attach(part)
elif BodyFile:
part = GetMIME(BodyFile, Body)
msg.attach(part)
elif Body:
msg.attach(MIMEText(Body))
for file in Attachments:
file = ExpandPath(file)
Verbose(' Attaching [file]')
if not os.path.exists(file):
Log('Attachment [file] does not exist')
continue
part = GetMIME(file)
part.add_header('Content-Disposition', 'attachment; filename="%s"' % os.path.basename(file))
msg.attach(part)
PrettyPrintList(list(msg.items()), Verbose=True)
Verbose(" ('Server', '[SMTPServer]')")
Log(" " + msg.as_string(), UseExpand=False, Verbose=True)
if TestMode:
Log(r'TestMode: skipping send')
return True
if UseService and PlatUtility.OpenActionEvent('EmailService', False):
EnsurePath(Globals.SendMailFolder)
fp = open(Expand(r'[SendMailFolder]\SendEmail.msg'), r'w')
fp.write(msg.as_string())
fp.close()
msgFile = CopyToUniqueFile(r'[SendMailFolder]\SendEmail.msg', r'[SendMailFolder]')
msgName = os.path.basename(msgFile)
emailData = JSON.load_from_file(Globals.SendMailJSON)
emailData[msgName] = {
'sender' : From,
'recipients' : To,
'contents' : msgFile,
}
JSON.save_to_file(Globals.SendMailJSON, emailData)
Verbose('Email saved for service')
PrettyPrintDict(emailData[msgName], Verbose=True)
if PlatUtility.SignalActionEvent('EmailService'):
return True
else:
JSON.delete_key_from_file(Globals.SendMailJSON, msgName)
Log('Sending email')
try:
smtp = smtplib.SMTP(Globals.SMTPServer)
if Globals.SMTPUserID:
smtp.ehlo()
smtp.starttls()
smtp.ehlo()
smtp.login(Globals.SMTPUserID, Password.decode(Globals.SMTPUserID, True))
smtp.sendmail(From, To, msg.as_string())
smtp.close()
Log('Email was sent')
return True
except:
ReportException()
return False
def apply_mtom(headers, envelope, params, paramvals):
'''Apply MTOM to a SOAP envelope, separating attachments into a
MIME multipart message.
References:
XOP http://www.w3.org/TR/xop10/
MTOM http://www.w3.org/TR/soap12-mtom/
http://www.w3.org/Submission/soap11mtom10/
@param headers Headers dictionary of the SOAP message that would
originally be sent.
@param envelope Iterable containing SOAP envelope string that would have
originally been sent.
@param params params attribute from the Message object used for the SOAP
@param paramvals values of the params, passed to Message.to_parent_element
@return tuple of length 2 with dictionary of headers and
string of body that can be sent with HTTPConnection
'''
# grab the XML element of the message in the SOAP body
envelope = ''.join(envelope)
soaptree = etree.fromstring(envelope)
soapbody = soaptree.find("{%s}Body" % rpclib.ns_soap_env)
message = None
for child in list(soapbody):
if child.tag == ("{%s}Fault" % rpclib.ns_soap_env):
return (headers, envelope)
else:
message = child
break
# Get additional parameters from original Content-Type
ctarray = []
for n, v in headers.items():
if n.lower() == 'content-type':
ctarray = v.split(';')
break
roottype = ctarray[0].strip()
rootparams = {}
for ctparam in ctarray[1:]:
n, v = ctparam.strip().split('=')
rootparams[n] = v.strip("\"'")
# Set up initial MIME parts.
mtompkg = MIMEMultipart('related',
boundary='?//<><>rpclib_MIME_boundary<>')
rootpkg = None
try:
rootpkg = MIMEApplication(envelope, 'xop+xml', encode_7or8bit)
except NameError:
rootpkg = MIMENonMultipart("application", "xop+xml")
rootpkg.set_payload(envelope)
encode_7or8bit(rootpkg)
# Set up multipart headers.
del (mtompkg['mime-version'])
mtompkg.set_param('start-info', roottype)
mtompkg.set_param('start', '<rpclibEnvelope>')
if 'SOAPAction' in headers:
mtompkg.add_header('SOAPAction', headers.get('SOAPAction'))
# Set up root SOAP part headers.
del (rootpkg['mime-version'])
rootpkg.add_header('Content-ID', '<rpclibEnvelope>')
for n, v in rootparams.items():
rootpkg.set_param(n, v)
rootpkg.set_param('type', roottype)
mtompkg.attach(rootpkg)
# Extract attachments from SOAP envelope.
for i in range(len(params)):
name, typ = params[i]
if typ == Attachment:
id = "rpclibAttachment_%s" % (len(mtompkg.get_payload()), )
param = message[i]
param.text = ""
incl = etree.SubElement(param, "{%s}Include" % rpclib.ns_xop)
incl.attrib["href"] = "cid:%s" % id
if paramvals[i].fileName and not paramvals[i].data:
paramvals[i].load_from_file()
data = paramvals[i].data
attachment = None
try:
attachment = MIMEApplication(data, _encoder=encode_7or8bit)
except NameError:
attachment = MIMENonMultipart("application", "octet-stream")
attachment.set_payload(data)
encode_7or8bit(attachment)
del (attachment['mime-version'])
attachment.add_header('Content-ID', '<%s>' % (id, ))
mtompkg.attach(attachment)
# Update SOAP envelope.
rootpkg.set_payload(etree.tostring(soaptree))
# extract body string from MIMEMultipart message
bound = '--%s' % (mtompkg.get_boundary(), )
marray = mtompkg.as_string().split(bound)
mtombody = bound
mtombody += bound.join(marray[1:])
# set Content-Length
mtompkg.add_header("Content-Length", str(len(mtombody)))
# extract dictionary of headers from MIMEMultipart message
mtomheaders = {}
for name, value in mtompkg.items():
mtomheaders[name] = value
if len(mtompkg.get_payload()) <= 1:
return (headers, envelope)
return (mtomheaders, [mtombody])
def apply_mtom(headers, envelope, params, paramvals):
"""
Apply MTOM to a SOAP envelope, separating attachments into a
MIME multipart message.
References:
XOP http://www.w3.org/TR/xop10/
MTOM http://www.w3.org/TR/soap12-mtom/
http://www.w3.org/Submission/soap11mtom10/
@param headers Headers dictionary of the SOAP message that would
originally be sent.
@param envelope SOAP envelope string that would have originally been sent.
@param params params attribute from the Message object used for the SOAP
@param paramvals values of the params, passed to Message.to_xml
@return tuple of length 2 with dictionary of headers and
string of body that can be sent with HTTPConnection
"""
# grab the XML element of the message in the SOAP body
soaptree = etree.fromstring(envelope)
soapbody = soaptree.find("{%s}Body" % soaplib.ns_soap_env)
message = None
for child in list(soapbody):
if child.tag != "%sFault" % (soaplib.ns_soap_env,):
message = child
break
# Get additional parameters from original Content-Type
ctarray = []
for n, v in headers.items():
if n.lower() == "content-type":
ctarray = v.split(";")
break
roottype = ctarray[0].strip()
rootparams = {}
for ctparam in ctarray[1:]:
n, v = ctparam.strip().split("=")
rootparams[n] = v.strip("\"'")
# Set up initial MIME parts.
mtompkg = MIMEMultipart("related", boundary="?//<><>soaplib_MIME_boundary<>")
rootpkg = None
try:
rootpkg = MIMEApplication(envelope, "xop+xml", encode_7or8bit)
except NameError:
rootpkg = MIMENonMultipart("application", "xop+xml")
rootpkg.set_payload(envelope)
encode_7or8bit(rootpkg)
# Set up multipart headers.
del (mtompkg["mime-version"])
mtompkg.set_param("start-info", roottype)
mtompkg.set_param("start", "<soaplibEnvelope>")
if "SOAPAction" in headers:
mtompkg.add_header("SOAPAction", headers.get("SOAPAction"))
# Set up root SOAP part headers.
del (rootpkg["mime-version"])
rootpkg.add_header("Content-ID", "<soaplibEnvelope>")
for n, v in rootparams.items():
rootpkg.set_param(n, v)
rootpkg.set_param("type", roottype)
mtompkg.attach(rootpkg)
# Extract attachments from SOAP envelope.
for i in range(len(params)):
name, typ = params[i]
if typ == Attachment:
id = "soaplibAttachment_%s" % (len(mtompkg.get_payload()),)
param = message[i]
param.text = ""
incl = etree.SubElement(param, "{%s}Include" % soaplib.ns_xop)
incl.attrib["href"] = "cid:%s" % id
if paramvals[i].fileName and not paramvals[i].data:
paramvals[i].load_from_file()
data = paramvals[i].data
attachment = None
try:
attachment = MIMEApplication(data, _encoder=encode_7or8bit)
except NameError:
attachment = MIMENonMultipart("application", "octet-stream")
attachment.set_payload(data)
encode_7or8bit(attachment)
del (attachment["mime-version"])
attachment.add_header("Content-ID", "<%s>" % (id,))
mtompkg.attach(attachment)
# Update SOAP envelope.
rootpkg.set_payload(etree.tostring(soaptree))
# extract body string from MIMEMultipart message
bound = "--%s" % (mtompkg.get_boundary(),)
marray = mtompkg.as_string().split(bound)
mtombody = bound
mtombody += bound.join(marray[1:])
# set Content-Length
mtompkg.add_header("Content-Length", str(len(mtombody)))
# extract dictionary of headers from MIMEMultipart message
mtomheaders = {}
for name, value in mtompkg.items():
mtomheaders[name] = value
if len(mtompkg.get_payload()) <= 1:
return (headers, envelope)
return (mtomheaders, mtombody)
def main():
try:
FORMAT = "%(asctime)-15s [%(levelname)-10s][%(process)s] %(message)s"
logging.basicConfig(filename=DEFAULTS['logfile'], level=allowed_log_levels[DEFAULTS['loglevel']], format=FORMAT)
logging.debug('Start processing...')
parser = email.parser.HeaderParser()
headers = parser.parse(sys.stdin)
from_addr = email.utils.parseaddr(headers.get('From'))
to_addr = email.utils.parseaddr(headers.get('To'))
xloop = email.utils.parseaddr(headers.get('X-loop'))
logging.debug(headers.items())
if xloop[1] == to_addr[1]:
logging.debug('X-loop detected. Exiting')
sys.exit(0)
if 'MAILER-DAEMON'.lower() in from_addr[1].lower():
logging.debug('MAILER-DAEMON detected. Exiting.')
sys.exit(0)
if ('Auto-submitted' or 'auto-replied' or 'auto-notified') in headers:
logging.debug('Autoreply detected. Exiting.')
sys.exit(0)
try:
db = MySQLdb.connect(host="localhost", user=DEFAULTS['dbuser'], passwd=DEFAULTS['dbpassword'],
db=DEFAULTS['db'], use_unicode=True, charset="utf8")
except Exception as e:
logging.exception('MySQL connection failure. Exiting.')
sys.exit(0)
c = db.cursor()
query = 'SELECT message FROM postfix_autoresponder WHERE email="{}" AND `from`<=NOW() AND `to`>=NOW()'.format(
to_addr[1])
c.execute(query)
logging.debug(query)
row = c.fetchone()
if row is None:
logging.debug('Autoresponder not found. Exiting.')
sys.exit(0)
msg = MIMEMultipart('alternative')
msg['From'] = to_addr[1]
msg['To'] = from_addr[1]
msg['Subject'] = 'Re: {}'.format(headers.get('Subject', ''))
msg['X-loop'] = to_addr[1]
msg['Auto-Submitted'] = 'auto-replied'
logging.debug(msg.items())
text = re.sub("<.*?>", "", row[0])
html = row[0]
part1 = MIMEText(text, 'plain', 'utf-8')
part2 = MIMEText(html, 'html', 'utf-8')
msg.attach(part1)
msg.attach(part2)
s = smtplib.SMTP('localhost')
s.sendmail(to_addr[1], from_addr[1], msg.as_string())
s.quit()
logging.debug('SENT to {}'.format(from_addr[1]))
logging.debug('Exiting.')
sys.exit(0)
except Exception as e:
logging.exception('main() failure')
sys.exit(0)
def submit(self):
"""Submit a query and parse the response.
@return: The data retrieved from api.php (a dict)
"""
paramstring = self.http_params()
while True:
action = self.params.get("action", "")
simulate = self._simulate(action)
if simulate:
return simulate
self.site.throttle(write=self.write)
uri = self.site.scriptpath() + "/api.php"
ssl = False
if self.site.family.name in config.available_ssl_project:
if action == "login" and config.use_SSL_onlogin:
ssl = True
elif config.use_SSL_always:
ssl = True
try:
if self.mime:
# construct a MIME message containing all API key/values
container = MIMEMultipart(_subtype='form-data')
for key in self.params:
# key "file" requires special treatment in a multipart
# message
if key == "file":
local_filename = self.params[key]
filetype = mimetypes.guess_type(local_filename)[0] \
or 'application/octet-stream'
file_content = file(local_filename, "rb").read()
submsg = MIMENonMultipart(*filetype.split("/"))
submsg.add_header("Content-disposition",
"form-data", name=key,
filename=local_filename)
submsg.set_payload(file_content)
else:
try:
self.params[key].encode("ascii")
keytype = ("text", "plain")
except UnicodeError:
keytype = ("application", "octet-stream")
submsg = MIMENonMultipart(*keytype)
submsg.add_header("Content-disposition", "form-data",
name=key)
submsg.set_payload(self.params[key])
container.attach(submsg)
# strip the headers to get the HTTP message body
body = container.as_string()
marker = "\n\n" # separates headers from body
eoh = body.find(marker)
body = body[ eoh + len(marker): ]
# retrieve the headers from the MIME object
mimehead = dict(container.items())
rawdata = http.request(self.site, uri, ssl, method="POST",
headers=mimehead, body=body)
else:
rawdata = http.request(self.site, uri, ssl, method="POST",
headers={'Content-Type':
'application/x-www-form-urlencoded'},
body=paramstring)
## import traceback
## traceback.print_stack()
## print rawdata
except Server504Error:
pywikibot.log(u"Caught HTTP 504 error; retrying")
self.wait()
continue
#TODO: what other exceptions can occur here?
except Exception, e:
# for any other error on the http request, wait and retry
pywikibot.error(traceback.format_exc())
pywikibot.log(u"%s, %s" % (uri, paramstring))
self.wait()
continue
if not isinstance(rawdata, unicode):
rawdata = rawdata.decode(self.site.encoding())
pywikibot.debug(u"API response received:\n" + rawdata, _logger)
if rawdata.startswith(u"unknown_action"):
raise APIError(rawdata[:14], rawdata[16:])
try:
result = json.loads(rawdata)
except ValueError:
# if the result isn't valid JSON, there must be a server
# problem. Wait a few seconds and try again
pywikibot.warning(
"Non-JSON response received from server %s; the server may be down."
% self.site)
pywikibot.debug(rawdata, _logger)
# there might also be an overflow, so try a smaller limit
for param in self.params:
if param.endswith("limit"):
value = self.params[param]
try:
self.params[param] = str(int(value) // 2)
pywikibot.output(u"Set %s = %s"
% (param, self.params[param]))
except:
pass
self.wait()
continue
if not result:
result = {}
if type(result) is not dict:
raise APIError("Unknown",
"Unable to process query response of type %s."
% type(result),
{'data': result})
if self['action'] == 'query':
if 'userinfo' in result.get('query', ()):
if hasattr(self.site, '_userinfo'):
self.site._userinfo.update(result['query']['userinfo'])
else:
self.site._userinfo = result['query']['userinfo']
status = self.site._loginstatus # save previous login status
if ( ("error" in result
and result["error"]["code"].endswith("limit"))
or (status >= 0
and self.site._userinfo['name']
!= self.site._username[status])):
# user is no longer logged in (session expired?)
# reset userinfo, then make user log in again
del self.site._userinfo
self.site._loginstatus = -1
if status < 0:
status = 0 # default to non-sysop login
self.site.login(status)
# retry the previous query
continue
if "warnings" in result:
modules = [k for k in result["warnings"] if k != "info"]
for mod in modules:
if '*' in result["warnings"][mod]:
text = result["warnings"][mod]['*']
elif 'html' in result["warnings"][mod]:
# Bugzilla 49978
text = result["warnings"][mod]['html']['*']
else:
# This is just a warning, we shouldn't raise an
# exception because of it
continue
pywikibot.warning(
u"API warning (%s): %s"
% (mod, text))
if "error" not in result:
return result
if "*" in result["error"]:
# help text returned
result['error']['help'] = result['error'].pop("*")
code = result["error"].pop("code", "Unknown")
info = result["error"].pop("info", None)
if code == "maxlag":
lag = lagpattern.search(info)
if lag:
pywikibot.log(
u"Pausing due to database lag: " + info)
self.site.throttle.lag(int(lag.group("lag")))
continue
if code in (u'internal_api_error_DBConnectionError', ):
self.wait()
continue
# raise error
try:
pywikibot.log(u"API Error: query=\n%s"
% pprint.pformat(self.params))
pywikibot.log(u" response=\n%s"
% result)
raise APIError(code, info, **result["error"])
except TypeError:
raise RuntimeError(result)
class Mdn(object):
"""Class for handling AS2 MDNs. Includes functions for both
parsing and building them.
"""
def __init__(self, mdn_mode=None, digest_alg=None, mdn_url=None):
self.message_id = None
self.orig_message_id = None
self.payload = None
self.mdn_mode = mdn_mode
self.digest_alg = digest_alg
self.mdn_url = mdn_url
@property
def content(self):
"""Function returns the body of the mdn message as a byte string"""
if self.payload:
message_bytes = mime_to_bytes(self.payload,
0).replace(b'\n', b'\r\n')
boundary = b'--' + self.payload.get_boundary().encode('utf-8')
temp = message_bytes.split(boundary)
temp.pop(0)
return boundary + boundary.join(temp)
else:
return ''
@property
def headers(self):
if self.payload:
return dict(self.payload.items())
else:
return {}
@property
def headers_str(self):
message_header = ''
if self.payload:
for k, v in self.headers.items():
message_header += '{}: {}\r\n'.format(k, v)
return message_header.encode('utf-8')
def build(self, message, status, detailed_status=None):
"""Function builds and signs an AS2 MDN message.
:param message: The received AS2 message for which this is an MDN.
:param status: The status of processing of the received AS2 message.
:param detailed_status:
The optional detailed status of processing of the received AS2
message. Used to give additional error info (default "None")
"""
# Generate message id using UUID 1 as it uses both hostname and time
self.message_id = email_utils.make_msgid().lstrip('<').rstrip('>')
self.orig_message_id = message.message_id
# Set up the message headers
mdn_headers = {
'AS2-Version': AS2_VERSION,
'ediint-features': EDIINT_FEATURES,
'Message-ID': '<{}>'.format(self.message_id),
'AS2-From': quote_as2name(message.headers.get('as2-to')),
'AS2-To': quote_as2name(message.headers.get('as2-from')),
'Date': email_utils.formatdate(localtime=True),
'user-agent': 'pyAS2 Open Source AS2 Software'
}
# Set the confirmation text message here
confirmation_text = MDN_CONFIRM_TEXT
# overwrite with organization specific message
if message.receiver and message.receiver.mdn_confirm_text:
confirmation_text = message.receiver.mdn_confirm_text
# overwrite with partner specific message
if message.sender and message.sender.mdn_confirm_text:
confirmation_text = message.sender.mdn_confirm_text
if status != 'processed':
confirmation_text = MDN_FAILED_TEXT
self.payload = MIMEMultipart('report',
report_type='disposition-notification')
# Create and attach the MDN Text Message
mdn_text = email_message.Message()
mdn_text.set_payload('%s\n' % confirmation_text)
mdn_text.set_type('text/plain')
del mdn_text['MIME-Version']
encoders.encode_7or8bit(mdn_text)
self.payload.attach(mdn_text)
# Create and attache the MDN Report Message
mdn_base = email_message.Message()
mdn_base.set_type('message/disposition-notification')
mdn_report = 'Reporting-UA: pyAS2 Open Source AS2 Software\n'
mdn_report += 'Original-Recipient: rfc822; {}\n'.format(
message.headers.get('as2-to'))
mdn_report += 'Final-Recipient: rfc822; {}\n'.format(
message.headers.get('as2-to'))
mdn_report += 'Original-Message-ID: <{}>\n'.format(message.message_id)
mdn_report += 'Disposition: automatic-action/' \
'MDN-sent-automatically; {}'.format(status)
if detailed_status:
mdn_report += ': {}'.format(detailed_status)
mdn_report += '\n'
if message.mic:
mdn_report += 'Received-content-MIC: {}, {}\n'.format(
message.mic.decode(), message.digest_alg)
mdn_base.set_payload(mdn_report)
del mdn_base['MIME-Version']
encoders.encode_7or8bit(mdn_base)
self.payload.attach(mdn_base)
logger.debug('MDN for message %s created:\n%s' %
(message.message_id, mdn_base.as_string()))
# Sign the MDN if it is requested by the sender
if message.headers.get('disposition-notification-options') and \
message.receiver and message.receiver.sign_key:
self.digest_alg = \
message.headers['disposition-notification-options'].split(
';')[-1].split(',')[-1].strip().replace('-', '').lower()
signed_mdn = MIMEMultipart('signed',
protocol="application/pkcs7-signature")
del signed_mdn['MIME-Version']
signed_mdn.attach(self.payload)
# Create the signature mime message
signature = email_message.Message()
signature.set_type('application/pkcs7-signature')
signature.set_param('name', 'smime.p7s')
signature.set_param('smime-type', 'signed-data')
signature.add_header('Content-Disposition',
'attachment',
filename='smime.p7s')
del signature['MIME-Version']
signature.set_payload(
sign_message(canonicalize(self.payload), self.digest_alg,
message.receiver.sign_key))
encoders.encode_base64(signature)
signed_mdn.set_param('micalg', self.digest_alg)
signed_mdn.attach(signature)
self.payload = signed_mdn
logger.debug('Signature for MDN %s created:\n%s' %
(message.message_id, signature.as_string()))
# Update the headers of the final payload and set message boundary
for k, v in mdn_headers.items():
if self.payload.get(k):
self.payload.replace_header(k, v)
else:
self.payload.add_header(k, v)
if self.payload.is_multipart():
self.payload.set_boundary(make_mime_boundary())
def parse(self, raw_content, find_message_cb):
"""Function parses the RAW AS2 MDN, verifies it and extracts the
processing status of the orginal AS2 message.
:param raw_content:
A byte string of the received HTTP headers followed by the body.
:param find_message_cb:
A callback the must returns the original Message Object. The
original message-id and original recipient AS2 ID are passed
as arguments to it.
:returns:
A two element tuple containing (status, detailed_status). The
status is a string indicating the status of the transaction. The
optional detailed_status gives additional information about the
processing status.
"""
status, detailed_status = None, None
self.payload = parse_mime(raw_content)
self.orig_message_id, orig_recipient = self.detect_mdn()
# Call the find message callback which should return a Message instance
orig_message = find_message_cb(self.orig_message_id, orig_recipient)
# Extract the headers and save it
mdn_headers = {}
for k, v in self.payload.items():
k = k.lower()
if k == 'message-id':
self.message_id = v.lstrip('<').rstrip('>')
mdn_headers[k] = v
if orig_message.receiver.mdn_digest_alg \
and self.payload.get_content_type() != 'multipart/signed':
status = 'failed/Failure'
detailed_status = 'Expected signed MDN but unsigned MDN returned'
return status, detailed_status
if self.payload.get_content_type() == 'multipart/signed':
message_boundary = ('--' + self.payload.get_boundary()).\
encode('utf-8')
# Extract the signature and the signed payload
signature = None
signature_types = [
'application/pkcs7-signature', 'application/x-pkcs7-signature'
]
for part in self.payload.walk():
if part.get_content_type() in signature_types:
signature = part.get_payload(decode=True)
elif part.get_content_type() == 'multipart/report':
self.payload = part
# Verify the message, first using raw message and if it fails
# then convert to canonical form and try again
mic_content = extract_first_part(raw_content, message_boundary)
verify_cert = orig_message.receiver.load_verify_cert()
try:
self.digest_alg = verify_message(mic_content, signature,
verify_cert)
except IntegrityError:
mic_content = canonicalize(self.payload)
self.digest_alg = verify_message(mic_content, signature,
verify_cert)
for part in self.payload.walk():
if part.get_content_type() == 'message/disposition-notification':
logger.debug('Found MDN report for message %s:\n%s' %
(orig_message.message_id, part.as_string()))
mdn = part.get_payload()[-1]
mdn_status = mdn['Disposition'].split(';').\
pop().strip().split(':')
status = mdn_status[0]
if status == 'processed':
mdn_mic = mdn.get('Received-Content-MIC', '').\
split(',')[0]
# TODO: Check MIC for all cases
if mdn_mic and orig_message.mic \
and mdn_mic != orig_message.mic.decode():
status = 'processed/warning'
detailed_status = 'Message Integrity check failed.'
else:
detailed_status = ' '.join(mdn_status[1:]).strip()
return status, detailed_status
def detect_mdn(self):
""" Function checks if the received raw message is an AS2 MDN or not.
:raises MDNNotFound: If the received payload is not an MDN then this
exception is raised.
:return:
A two element tuple containing (message_id, message_recipient). The
message_id is the original AS2 message id and the message_recipient
is the original AS2 message recipient.
"""
mdn_message = None
if self.payload.get_content_type() == 'multipart/report':
mdn_message = self.payload
elif self.payload.get_content_type() == 'multipart/signed':
for part in self.payload.walk():
if part.get_content_type() == 'multipart/report':
mdn_message = self.payload
if not mdn_message:
raise MDNNotFound('No MDN found in the received message')
message_id, message_recipient = None, None
for part in mdn_message.walk():
if part.get_content_type() == 'message/disposition-notification':
mdn = part.get_payload()[0]
message_id = mdn.get('Original-Message-ID').strip('<>')
message_recipient = mdn.get('Original-Recipient').\
split(';')[1].strip()
return message_id, message_recipient
