def _get_message_parts(self, message):
"""
Extracts all parts from a message and returns them in a dict
Code from the mail_gateway module
"""
if isinstance(message, xmlrpclib.Binary):
message = str(message.data)
# Warning: message_from_string doesn't always work correctly on unicode,
# we must use utf-8 strings here :-(
if isinstance(message, unicode):
message = message.encode('utf-8')
msg_txt = email.message_from_string(message)
msg = {}
fields = msg_txt.keys()
if 'Subject' in fields:
msg['subject'] = self._decode_header(msg_txt.get('Subject'))
if 'Content-Type' in fields:
msg['content-type'] = msg_txt.get('Content-Type')
if 'From' in fields:
msg['from'] = self._decode_header(msg_txt.get('From'))
if 'Delivered-To' in fields:
msg['to'] = self._decode_header(msg_txt.get('Delivered-To'))
if 'CC' in fields:
msg['cc'] = self._decode_header(msg_txt.get('CC'))
if 'Reply-to' in fields:
msg['reply-to'] = self._decode_header(msg_txt.get('Reply-To'))
if 'Date' in fields:
msg['date'] = self._decode_header(msg_txt.get('Date'))
if 'Content-Transfer-Encoding' in fields:
msg['encoding'] = msg_txt.get('Content-Transfer-Encoding')
if 'References' in fields:
msg['references'] = msg_txt.get('References')
if 'In-Reply-To' in fields:
msg['in-reply-to'] = msg_txt.get('In-Reply-To')
if not msg_txt.is_multipart() or 'text/plain' in msg.get('Content-Type', ''):
encoding = msg_txt.get_content_charset()
body = msg_txt.get_payload(decode=True)
if 'text/html' in msg_txt.get('Content-Type', ''):
body = tools.html2plaintext(body)
msg['body'] = tools.ustr(body, encoding)
return msg
def send_get_email_dict(self, cr, uid, mail, partner=None, context=None):
""" Return a dictionary for specific email values, depending on a
partner, or generic to the whole recipients given by mail.email_to.
:param browse_record mail: mail.mail browse_record
:param browse_record partner: specific recipient partner
"""
body = self.send_get_mail_body(cr, uid, mail, partner=partner, context=context)
subject = self.send_get_mail_subject(cr, uid, mail, partner=partner, context=context)
body_alternative = tools.html2plaintext(body)
email_to = [partner.email] if partner else tools.email_split(mail.email_to)
return {"body": body, "body_alternative": body_alternative, "subject": subject, "email_to": email_to}
def message_new(self, cr, uid, msg, custom_values=None, context=None):
""" Overrides mail_thread message_new that is called by the mailgateway
through message_process.
This override updates the document according to the email.
"""
if custom_values is None: custom_values = {}
desc = html2plaintext(msg.get('body')) if msg.get('body') else ''
custom_values.update({
'name': msg.get('subject') or _("No Subject"),
'description': desc,
'email_from': msg.get('from'),
'email_cc': msg.get('cc'),
'user_id': False,
})
return super(crm_helpdesk,self).message_new(cr, uid, msg, custom_values=custom_values, context=context)
def message_new(self, cr, uid, msg, custom_values=None, context=None):
""" Overrides mail_thread message_new that is called by the mailgateway
through message_process.
This override updates the document according to the email.
"""
if custom_values is None: custom_values = {}
if context is None: context = {}
context['state_to'] = 'draft'
desc = html2plaintext(msg.get('body')) if msg.get('body') else ''
custom_values.update({
'name': msg.get('subject') or _("No Subject"),
'description': desc,
'email_from': msg.get('from'),
'email_cc': msg.get('cc'),
'user_id': False,
})
if msg.get('priority'):
custom_values['priority'] = msg.get('priority')
res_id = super(project_issue, self).message_new(cr, uid, msg, custom_values=custom_values, context=context)
# self.convert_to_bug(cr, uid, [res_id], context=context)
return res_id
def psrDist(psr):
name = psr.split()
if len(name) == 2:
if name[0] == 'PSR':
psr = name[1]
else:
raise 'Can only deal with radio pulsar.'
if PSRDict.has_key(psr):
url = 'https://www.nrl.navy.mil/rsd/RORF/ne2001/pulsar.cgi'
values = {'pulsar': PSRDict[psr], "frequency": 1.}
data = urllib.urlencode(values)
req = urllib2.Request(url, data)
gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1) # Only for gangstars
response = urllib2.urlopen(req, context=gcontext)
the_page = response.read()
text = html2plaintext(the_page, encoding='UTF-8')
p = re.compile('D = (.+) kpc\\n\((.+),(.+)\)', re.U)
m = p.search(text)
dist = [float(D) for D in m.groups(0)]
return (dist[0], dist[1] - dist[0], dist[2] - dist[0])
else:
#raise "%s's information is not available" % psr
dist = Qatnf(psr, ['Dist'])['Dist']
return (float(dist[0]), dist[1])
def psrDist(psr):
name = psr.split()
if len(name) == 2:
if name[0] == "PSR":
psr = name[1]
else:
raise "Can only deal with radio pulsar."
if PSRDict.has_key(psr):
url = "http://rsd-www.nrl.navy.mil/cgi-bin/pulsar.cgi"
values = {"pulsar": PSRDict[psr], "frequency": 1.0}
data = urllib.urlencode(values)
req = urllib2.Request(url, data)
response = urllib2.urlopen(req)
the_page = response.read()
text = html2plaintext(the_page, encoding="UTF-8")
# print text
p = re.compile("D = (.+) kpc\\n\((.+),(.+)\)", re.U)
m = p.search(text)
dist = [float(D) for D in m.groups(0)]
return (dist[0], dist[1] - dist[0], dist[2] - dist[0])
else:
# raise "%s's information is not available" % psr
dist = Qatnf(psr, ["Dist"])["Dist"]
return (float(dist[0]), dist[1])
def parse_message(self, message):
#TOCHECK: put this function in mailgateway module
if isinstance(message, unicode):
message = message.encode('utf-8')
msg_txt = email.message_from_string(message)
message_id = msg_txt.get('message-id', False)
msg = {}
fields = msg_txt.keys()
msg['id'] = message_id
msg['message-id'] = message_id
if 'Subject' in fields:
msg['subject'] = self._decode_header(msg_txt.get('Subject'))
if 'Content-Type' in fields:
msg['content-type'] = msg_txt.get('Content-Type')
if 'From' in fields:
msg['from'] = self._decode_header(msg_txt.get('From'))
if 'To' in fields:
msg['to'] = self._decode_header(msg_txt.get('To'))
else:
msg['to'] = self._decode_header(msg_txt.get('Delivered-To'))
if 'CC' in fields:
msg['cc'] = self._decode_header(msg_txt.get('CC'))
if 'Reply-to' in fields:
msg['reply'] = self._decode_header(msg_txt.get('Reply-To'))
if 'Date' in fields:
date = self._decode_header(msg_txt.get('Date'))
msg['date'] = dateutil.parser.parse(date).strftime("%Y-%m-%d %H:%M:%S")
if 'Content-Transfer-Encoding' in fields:
msg['encoding'] = msg_txt.get('Content-Transfer-Encoding')
if 'References' in fields:
msg['references'] = msg_txt.get('References')
if 'In-Reply-To' in fields:
msg['in-reply-to'] = msg_txt.get('In-Reply-To')
if 'X-Priority' in fields:
msg['priority'] = msg_txt.get('X-Priority', '3 (Normal)').split(' ')[0]
if not msg_txt.is_multipart() or 'text/plain' in msg.get('Content-Type', ''):
encoding = msg_txt.get_content_charset()
body = msg_txt.get_payload(decode=True)
msg['body'] = tools.ustr(body, encoding)
attachments = {}
has_plain_text = False
if msg_txt.is_multipart() or 'multipart/alternative' in msg.get('content-type', ''):
body = ""
for part in msg_txt.walk():
if part.get_content_maintype() == 'multipart':
continue
encoding = part.get_content_charset()
filename = part.get_filename()
if part.get_content_maintype()=='text':
content = part.get_payload(decode=True)
if filename:
attachments[filename] = content
elif not has_plain_text:
# main content parts should have 'text' maintype
# and no filename. we ignore the html part if
# there is already a plaintext part without filename,
# because presumably these are alternatives.
content = tools.ustr(content, encoding)
if part.get_content_subtype() == 'html':
body = tools.ustr(tools.html2plaintext(content))
elif part.get_content_subtype() == 'plain':
body = content
has_plain_text = True
elif part.get_content_maintype() in ('application', 'image') \
or part.get_content_subtype() in ('octet-stream'):
if filename :
attachments[filename] = part.get_payload(decode=True)
else:
res = part.get_payload(decode=True)
body += tools.ustr(res, encoding)
msg['body'] = body
msg['attachments'] = attachments
return msg
def process_email(self, cr, uid, model, message, custom_values=None, attach=True, context=None):
"""This function Processes email and create record for given OpenERP model
@param self: The object pointer
@param cr: the current row, from the database cursor,
@param uid: the current user’s ID for security checks,
@param model: OpenObject Model
@param message: Email details, passed as a string or an xmlrpclib.Binary
@param attach: Email attachments
@param context: A standard dictionary for contextual values"""
# extract message bytes, we are forced to pass the message as binary because
# we don't know its encoding until we parse its headers and hence can't
# convert it to utf-8 for transport between the mailgate script and here.
if isinstance(message, xmlrpclib.Binary):
message = str(message.data)
if context is None:
context = {}
if custom_values is None or not isinstance(custom_values, dict):
custom_values = {}
model_pool = self.pool.get(model)
res_id = False
# Create New Record into particular model
def create_record(msg):
att_ids = []
if hasattr(model_pool, 'message_new'):
res_id,att_ids = model_pool.message_new(cr, uid, msg, context=context)
if custom_values:
model_pool.write(cr, uid, [res_id], custom_values, context=context)
else:
data = {
'name': msg.get('subject'),
'email_from': msg.get('from'),
'email_cc': msg.get('cc'),
'user_id': False,
'description': msg.get('body'),
'state' : 'draft',
}
data.update(self.get_partner(cr, uid, msg.get('from'), context=context))
res_id = model_pool.create(cr, uid, data, context=context)
if attach:
for attachment in msg.get('attachments', []):
data_attach = {
'name': attachment,
'datas': binascii.b2a_base64(str(attachments.get(attachment))),
'datas_fname': attachment,
'description': 'Mail attachment',
'res_model': model,
'res_id': res_id,
}
att_ids.append(self.pool.get('ir.attachment').create(cr, uid, data_attach))
return res_id, att_ids
# Warning: message_from_string doesn't always work correctly on unicode,
# we must use utf-8 strings here :-(
if isinstance(message, unicode):
message = message.encode('utf-8')
msg_txt = email.message_from_string(message)
message_id = msg_txt.get('message-id', False)
msg = {}
if not message_id:
# Very unusual situation, be we should be fault-tolerant here
message_id = time.time()
msg_txt['message-id'] = message_id
_logger.info('Message without message-id, generating a random one: %s', message_id)
fields = msg_txt.keys()
msg['id'] = message_id
msg['message-id'] = message_id
if 'Subject' in fields:
msg['subject'] = self._decode_header(msg_txt.get('Subject'))
if 'Content-Type' in fields:
msg['content-type'] = msg_txt.get('Content-Type')
if 'From' in fields:
msg['from'] = self._decode_header(msg_txt.get('From'))
if 'Delivered-To' in fields:
msg['to'] = self._decode_header(msg_txt.get('Delivered-To'))
if 'CC' in fields:
msg['cc'] = self._decode_header(msg_txt.get('CC'))
if 'Reply-to' in fields:
msg['reply'] = self._decode_header(msg_txt.get('Reply-To'))
if 'Date' in fields:
msg['date'] = self._decode_header(msg_txt.get('Date'))
if 'Content-Transfer-Encoding' in fields:
msg['encoding'] = msg_txt.get('Content-Transfer-Encoding')
if 'References' in fields:
msg['references'] = msg_txt.get('References')
if 'In-Reply-To' in fields:
msg['in-reply-to'] = msg_txt.get('In-Reply-To')
if 'X-Priority' in fields:
msg['priority'] = msg_txt.get('X-Priority', '3 (Normal)').split(' ')[0]
if not msg_txt.is_multipart() or 'text/plain' in msg.get('Content-Type', ''):
encoding = msg_txt.get_content_charset()
body = msg_txt.get_payload(decode=True)
if 'text/html' in msg_txt.get('Content-Type', ''):
body = tools.html2plaintext(body)
msg['body'] = tools.ustr(body, encoding)
attachments = {}
has_plain_text = False
if msg_txt.is_multipart() or 'multipart/alternative' in msg.get('content-type', ''):
body = ""
for part in msg_txt.walk():
if part.get_content_maintype() == 'multipart':
continue
encoding = part.get_content_charset()
filename = part.get_filename()
if part.get_content_maintype()=='text':
content = part.get_payload(decode=True)
if filename:
attachments[filename] = content
elif not has_plain_text:
# main content parts should have 'text' maintype
# and no filename. we ignore the html part if
# there is already a plaintext part without filename,
# because presumably these are alternatives.
content = tools.ustr(content, encoding)
if part.get_content_subtype() == 'html':
body = tools.ustr(tools.html2plaintext(content))
elif part.get_content_subtype() == 'plain':
body = content
has_plain_text = True
elif part.get_content_maintype() in ('application', 'image') \
or part.get_content_subtype() in ('octet-stream'):
if filename :
attachments[filename] = part.get_payload(decode=True)
else:
res = part.get_payload(decode=True)
body += tools.ustr(res, encoding)
msg['body'] = body
msg['attachments'] = attachments
res_ids = []
attachment_ids = []
new_res_id = False
if msg.get('references') or msg.get('in-reply-to'):
references = msg.get('references') or msg.get('in-reply-to')
if '\r\n' in references:
references = references.split('\r\n')
else:
references = references.split(' ')
for ref in references:
ref = ref.strip()
res_id = tools.misc.reference_re.search(ref)
if res_id:
res_id = res_id.group(1)
else:
res_id = tools.misc.res_re.search(msg['subject'])
if res_id:
res_id = res_id.group(1)
if res_id:
res_id = int(res_id)
model_pool = self.pool.get(model)
if model_pool.exists(cr, uid, res_id):
res_ids.append(res_id)
if hasattr(model_pool, 'message_update'):
model_pool.message_update(cr, uid, [res_id], {}, msg, context=context)
else:
raise NotImplementedError('model %s does not support updating records, mailgate API method message_update() is missing'%model)
if not len(res_ids):
new_res_id, attachment_ids = create_record(msg)
res_ids = [new_res_id]
# Store messages
context.update({'model' : model})
if hasattr(model_pool, 'history'):
model_pool.history(cr, uid, res_ids, _('receive'), history=True,
subject = msg.get('subject'),
email = msg.get('to'),
details = msg.get('body'),
email_from = msg.get('from'),
email_cc = msg.get('cc'),
message_id = msg.get('message-id'),
references = msg.get('references', False) or msg.get('in-reply-to', False),
attach = attachment_ids or attachments.items(),
email_date = msg.get('date'),
context = context)
else:
self.history(cr, uid, model, res_ids, msg, attachment_ids, context=context)
self.email_forward(cr, uid, model, res_ids, msg_txt)
return new_res_id
def parse_message(self, message, save_original=False):
"""Parses a string or email.message.Message representing an
RFC-2822 email, and returns a generic dict holding the
message details.
:param message: the message to parse
:type message: email.message.Message | string | unicode
:param bool save_original: whether the returned dict
should include an ``original`` entry with the base64
encoded source of the message.
:rtype: dict
:return: A dict with the following structure, where each
field may not be present if missing in original
message::
{ 'message-id': msg_id,
'subject': subject,
'from': from,
'to': to,
'cc': cc,
'headers' : { 'X-Mailer': mailer,
#.. all X- headers...
},
'subtype': msg_mime_subtype,
'body_text': plaintext_body
'body_html': html_body,
'attachments': [('file1', 'bytes'),
('file2', 'bytes') }
# ...
'original': source_of_email,
}
"""
msg_txt = message
if isinstance(message, str):
msg_txt = email.message_from_string(message)
# Warning: message_from_string doesn't always work correctly on unicode,
# we must use utf-8 strings here :-(
if isinstance(message, unicode):
message = message.encode('utf-8')
msg_txt = email.message_from_string(message)
message_id = msg_txt.get('message-id', False)
msg = {}
if save_original:
# save original, we need to be able to read the original email sometimes
msg['original'] = message.as_string() if isinstance(message, Message) \
else message
msg['original'] = base64.b64encode(
msg['original']) # binary fields are b64
if not message_id:
# Very unusual situation, be we should be fault-tolerant here
message_id = time.time()
msg_txt['message-id'] = message_id
_logger.info(
'Parsing Message without message-id, generating a random one: %s',
message_id)
fields = msg_txt.keys()
msg['id'] = message_id
msg['message-id'] = message_id
if 'Subject' in fields:
msg['subject'] = decode(msg_txt.get('Subject'))
if 'Content-Type' in fields:
msg['content-type'] = msg_txt.get('Content-Type')
if 'From' in fields:
msg['from'] = decode(msg_txt.get('From') or msg_txt.get_unixfrom())
if 'To' in fields:
msg['to'] = decode(msg_txt.get('To'))
if 'Delivered-To' in fields:
msg['to'] = decode(msg_txt.get('Delivered-To'))
if 'CC' in fields:
msg['cc'] = decode(msg_txt.get('CC'))
if 'Cc' in fields:
msg['cc'] = decode(msg_txt.get('Cc'))
if 'Reply-To' in fields:
msg['reply'] = decode(msg_txt.get('Reply-To'))
if 'Date' in fields:
try:
date_hdr = decode(msg_txt.get('Date'))
parsed_date = dateutil.parser.parse(date_hdr, fuzzy=True)
if parsed_date.utcoffset() is None:
# naive datetime, so we arbitrarily decide to make it
# UTC, there's no better choice. Should not happen,
# as RFC2822 requires timezone offset in Date headers.
stored_date = parsed_date.replace(tzinfo=pytz.utc)
else:
stored_date = parsed_date.astimezone(pytz.utc)
except Exception:
_logger.warning(
'Failed to parse Date header %r in incoming mail '
'with message-id %r, assuming current date/time.',
msg_txt.get('Date'), message_id)
stored_date = datetime.datetime.now()
msg['date'] = stored_date.strftime("%Y-%m-%d %H:%M:%S")
if 'Content-Transfer-Encoding' in fields:
msg['encoding'] = msg_txt.get('Content-Transfer-Encoding')
if 'References' in fields:
msg['references'] = msg_txt.get('References')
if 'In-Reply-To' in fields:
msg['in-reply-to'] = msg_txt.get('In-Reply-To')
msg['headers'] = {}
msg['subtype'] = 'plain'
for item in msg_txt.items():
if item[0].startswith('X-'):
msg['headers'].update({item[0]: item[1]})
if not msg_txt.is_multipart() or 'text/plain' in msg.get(
'content-type', ''):
encoding = msg_txt.get_content_charset()
body = tools.ustr(msg_txt.get_payload(decode=True),
encoding,
errors='replace')
if 'text/html' in msg.get('content-type', ''):
msg['body_html'] = body
msg['subtype'] = 'html'
body = tools.html2plaintext(body)
msg['body_text'] = tools.ustr(body, encoding, errors='replace')
attachments = []
if msg_txt.is_multipart() or 'multipart/alternative' in msg.get(
'content-type', ''):
body = ""
if 'multipart/alternative' in msg.get('content-type', ''):
msg['subtype'] = 'alternative'
else:
msg['subtype'] = 'mixed'
for part in msg_txt.walk():
if part.get_content_maintype() == 'multipart':
continue
encoding = part.get_content_charset()
filename = part.get_filename()
if part.get_content_maintype() == 'text':
content = part.get_payload(decode=True)
if filename:
attachments.append((filename, content))
content = tools.ustr(content, encoding, errors='replace')
if part.get_content_subtype() == 'html':
msg['body_html'] = content
msg['subtype'] = 'html' # html version prevails
body = tools.ustr(tools.html2plaintext(content))
body = body.replace(' ', '')
elif part.get_content_subtype() == 'plain':
body = content
elif part.get_content_maintype() in ('application', 'image'):
if filename:
attachments.append(
(filename, part.get_payload(decode=True)))
else:
res = part.get_payload(decode=True)
body += tools.ustr(res, encoding, errors='replace')
msg['body_text'] = body
msg['attachments'] = attachments
# for backwards compatibility:
msg['body'] = msg['body_text']
msg['sub_type'] = msg['subtype'] or 'plain'
return msg
def process_email(self,
cr,
uid,
model,
message,
custom_values=None,
attach=True,
context=None):
"""This function Processes email and create record for given OpenERP model
@param self: The object pointer
@param cr: the current row, from the database cursor,
@param uid: the current user’s ID for security checks,
@param model: OpenObject Model
@param message: Email details, passed as a string or an xmlrpclib.Binary
@param attach: Email attachments
@param context: A standard dictionary for contextual values"""
# extract message bytes, we are forced to pass the message as binary because
# we don't know its encoding until we parse its headers and hence can't
# convert it to utf-8 for transport between the mailgate script and here.
if isinstance(message, xmlrpclib.Binary):
message = str(message.data)
if context is None:
context = {}
if custom_values is None or not isinstance(custom_values, dict):
custom_values = {}
model_pool = self.pool.get(model)
res_id = False
# Create New Record into particular model
def create_record(msg):
att_ids = []
if hasattr(model_pool, 'message_new'):
res_id, att_ids = model_pool.message_new(cr,
uid,
msg,
context=context)
if custom_values:
model_pool.write(cr,
uid, [res_id],
custom_values,
context=context)
else:
data = {
'name': msg.get('subject'),
'email_from': msg.get('from'),
'email_cc': msg.get('cc'),
'user_id': False,
'description': msg.get('body'),
'state': 'draft',
}
data.update(
self.get_partner(cr, uid, msg.get('from'),
context=context))
res_id = model_pool.create(cr, uid, data, context=context)
if attach:
for attachment in msg.get('attachments', []):
data_attach = {
'name':
attachment,
'datas':
binascii.b2a_base64(
str(attachments.get(attachment))),
'datas_fname':
attachment,
'description':
'Mail attachment',
'res_model':
model,
'res_id':
res_id,
}
att_ids.append(
self.pool.get('ir.attachment').create(
cr, uid, data_attach))
return res_id, att_ids
# Warning: message_from_string doesn't always work correctly on unicode,
# we must use utf-8 strings here :-(
if isinstance(message, unicode):
message = message.encode('utf-8')
msg_txt = email.message_from_string(message)
message_id = msg_txt.get('message-id', False)
msg = {}
if not message_id:
# Very unusual situation, be we should be fault-tolerant here
message_id = time.time()
msg_txt['message-id'] = message_id
_logger.info(
'Message without message-id, generating a random one: %s',
message_id)
fields = msg_txt.keys()
msg['id'] = message_id
msg['message-id'] = message_id
if 'Subject' in fields:
msg['subject'] = self._decode_header(msg_txt.get('Subject'))
if 'Content-Type' in fields:
msg['content-type'] = msg_txt.get('Content-Type')
if 'From' in fields:
msg['from'] = self._decode_header(msg_txt.get('From'))
if 'Delivered-To' in fields:
msg['to'] = self._decode_header(msg_txt.get('Delivered-To'))
if 'CC' in fields:
msg['cc'] = self._decode_header(msg_txt.get('CC'))
if 'Reply-to' in fields:
msg['reply'] = self._decode_header(msg_txt.get('Reply-To'))
if 'Date' in fields:
msg['date'] = self._decode_header(msg_txt.get('Date'))
if 'Content-Transfer-Encoding' in fields:
msg['encoding'] = msg_txt.get('Content-Transfer-Encoding')
if 'References' in fields:
msg['references'] = msg_txt.get('References')
if 'In-Reply-To' in fields:
msg['in-reply-to'] = msg_txt.get('In-Reply-To')
if 'X-Priority' in fields:
msg['priority'] = msg_txt.get('X-Priority',
'3 (Normal)').split(' ')[0]
if not msg_txt.is_multipart() or 'text/plain' in msg.get(
'Content-Type', ''):
encoding = msg_txt.get_content_charset()
body = msg_txt.get_payload(decode=True)
if 'text/html' in msg_txt.get('Content-Type', ''):
body = tools.html2plaintext(body)
msg['body'] = tools.ustr(body, encoding)
attachments = {}
has_plain_text = False
if msg_txt.is_multipart() or 'multipart/alternative' in msg.get(
'content-type', ''):
body = ""
for part in msg_txt.walk():
if part.get_content_maintype() == 'multipart':
continue
encoding = part.get_content_charset()
filename = part.get_filename()
if part.get_content_maintype() == 'text':
content = part.get_payload(decode=True)
if filename:
attachments[filename] = content
elif not has_plain_text:
# main content parts should have 'text' maintype
# and no filename. we ignore the html part if
# there is already a plaintext part without filename,
# because presumably these are alternatives.
content = tools.ustr(content, encoding)
if part.get_content_subtype() == 'html':
body = tools.ustr(tools.html2plaintext(content))
elif part.get_content_subtype() == 'plain':
body = content
has_plain_text = True
elif part.get_content_maintype() in ('application', 'image'):
if filename:
attachments[filename] = part.get_payload(decode=True)
else:
res = part.get_payload(decode=True)
body += tools.ustr(res, encoding)
msg['body'] = body
msg['attachments'] = attachments
res_ids = []
attachment_ids = []
new_res_id = False
if msg.get('references') or msg.get('in-reply-to'):
references = msg.get('references') or msg.get('in-reply-to')
if '\r\n' in references:
references = references.split('\r\n')
else:
references = references.split(' ')
for ref in references:
ref = ref.strip()
res_id = tools.misc.reference_re.search(ref)
if res_id:
res_id = res_id.group(1)
else:
res_id = tools.misc.res_re.search(msg['subject'])
if res_id:
res_id = res_id.group(1)
if res_id:
res_id = int(res_id)
model_pool = self.pool.get(model)
if model_pool.exists(cr, uid, res_id):
res_ids.append(res_id)
if hasattr(model_pool, 'message_update'):
model_pool.message_update(cr,
uid, [res_id], {},
msg,
context=context)
else:
raise NotImplementedError(
'model %s does not support updating records, mailgate API method message_update() is missing'
% model)
if not len(res_ids):
new_res_id, attachment_ids = create_record(msg)
res_ids = [new_res_id]
# Store messages
context.update({'model': model})
if hasattr(model_pool, 'history'):
model_pool.history(cr,
uid,
res_ids,
_('receive'),
history=True,
subject=msg.get('subject'),
email=msg.get('to'),
details=msg.get('body'),
email_from=msg.get('from'),
email_cc=msg.get('cc'),
message_id=msg.get('message-id'),
references=msg.get('references', False)
or msg.get('in-reply-to', False),
attach=attachment_ids or attachments.items(),
email_date=msg.get('date'),
context=context)
else:
self.history(cr,
uid,
model,
res_ids,
msg,
attachment_ids,
context=context)
self.email_forward(cr, uid, model, res_ids, msg_txt)
return new_res_id
def process_email(self, cr, uid, model, message, custom_values=None, attach=True, context=None):
"""This function Processes email and create record for given OpenERP model
@param self: The object pointer
@param cr: the current row, from the database cursor,
@param uid: the current user’s ID for security checks,
@param model: OpenObject Model
@param message: Email details, passed as a string or an xmlrpclib.Binary
@param attach: Email attachments
@param context: A standard dictionary for contextual values"""
# extract message bytes, we are forced to pass the message as binary because
# we don't know its encoding until we parse its headers and hence can't
# convert it to utf-8 for transport between the mailgate script and here.
if isinstance(message, xmlrpclib.Binary):
message = str(message.data)
if context is None:
context = {}
if custom_values is None or not isinstance(custom_values, dict):
custom_values = {}
model_pool = self.pool.get(model)
res_id = False
# Create New Record into particular model
def create_record(msg):
att_ids = []
if hasattr(model_pool, "message_new"):
res_id = model_pool.message_new(cr, uid, msg, context=context)
if custom_values:
model_pool.write(cr, uid, [res_id], custom_values, context=context)
else:
data = {
"name": msg.get("subject"),
"email_from": msg.get("from"),
"email_cc": msg.get("cc"),
"user_id": False,
"description": msg.get("body"),
"state": "draft",
}
data.update(self.get_partner(cr, uid, msg.get("from"), context=context))
res_id = model_pool.create(cr, uid, data, context=context)
if attach:
for attachment in msg.get("attachments", []):
data_attach = {
"name": attachment,
"datas": binascii.b2a_base64(str(attachments.get(attachment))),
"datas_fname": attachment,
"description": "Mail attachment",
"res_model": model,
"res_id": res_id,
}
att_ids.append(self.pool.get("ir.attachment").create(cr, uid, data_attach))
return res_id, att_ids
# Warning: message_from_string doesn't always work correctly on unicode,
# we must use utf-8 strings here :-(
if isinstance(message, unicode):
message = message.encode("utf-8")
msg_txt = email.message_from_string(message)
message_id = msg_txt.get("message-id", False)
msg = {}
if not message_id:
# Very unusual situation, be we should be fault-tolerant here
message_id = time.time()
msg_txt["message-id"] = message_id
_logger.info("Message without message-id, generating a random one: %s", message_id)
fields = msg_txt.keys()
msg["id"] = message_id
msg["message-id"] = message_id
if "Subject" in fields:
msg["subject"] = self._decode_header(msg_txt.get("Subject"))
if "Content-Type" in fields:
msg["content-type"] = msg_txt.get("Content-Type")
if "From" in fields:
msg["from"] = self._decode_header(msg_txt.get("From"))
if "Delivered-To" in fields:
msg["to"] = self._decode_header(msg_txt.get("Delivered-To"))
if "CC" in fields:
msg["cc"] = self._decode_header(msg_txt.get("CC"))
if "Reply-to" in fields:
msg["reply"] = self._decode_header(msg_txt.get("Reply-To"))
if "Date" in fields:
msg["date"] = self._decode_header(msg_txt.get("Date"))
if "Content-Transfer-Encoding" in fields:
msg["encoding"] = msg_txt.get("Content-Transfer-Encoding")
if "References" in fields:
msg["references"] = msg_txt.get("References")
if "In-Reply-To" in fields:
msg["in-reply-to"] = msg_txt.get("In-Reply-To")
if "X-Priority" in fields:
msg["priority"] = msg_txt.get("X-Priority", "3 (Normal)").split(" ")[0]
if not msg_txt.is_multipart() or "text/plain" in msg.get("Content-Type", ""):
encoding = msg_txt.get_content_charset()
body = msg_txt.get_payload(decode=True)
if "text/html" in msg_txt.get("Content-Type", ""):
body = tools.html2plaintext(body)
msg["body"] = tools.ustr(body, encoding)
attachments = {}
has_plain_text = False
if msg_txt.is_multipart() or "multipart/alternative" in msg.get("content-type", ""):
body = ""
for part in msg_txt.walk():
if part.get_content_maintype() == "multipart":
continue
encoding = part.get_content_charset()
filename = part.get_filename()
if part.get_content_maintype() == "text":
content = part.get_payload(decode=True)
if filename:
attachments[filename] = content
elif not has_plain_text:
# main content parts should have 'text' maintype
# and no filename. we ignore the html part if
# there is already a plaintext part without filename,
# because presumably these are alternatives.
content = tools.ustr(content, encoding)
if part.get_content_subtype() == "html":
body = tools.ustr(tools.html2plaintext(content))
elif part.get_content_subtype() == "plain":
body = content
has_plain_text = True
elif part.get_content_maintype() in ("application", "image"):
if filename and attach:
attachments[filename] = part.get_payload(decode=True)
else:
res = part.get_payload(decode=True)
body += tools.ustr(res, encoding)
msg["body"] = body
msg["attachments"] = attachments
res_ids = []
attachment_ids = []
new_res_id = False
if msg.get("references") or msg.get("in-reply-to"):
references = msg.get("references") or msg.get("in-reply-to")
if "\r\n" in references:
references = references.split("\r\n")
else:
references = references.split(" ")
for ref in references:
ref = ref.strip()
res_id = tools.misc.reference_re.search(ref)
if res_id:
res_id = res_id.group(1)
else:
res_id = tools.misc.res_re.search(msg["subject"])
if res_id:
res_id = res_id.group(1)
if res_id:
res_id = int(res_id)
model_pool = self.pool.get(model)
if model_pool.exists(cr, uid, res_id):
res_ids.append(res_id)
if hasattr(model_pool, "message_update"):
model_pool.message_update(cr, uid, [res_id], {}, msg, context=context)
else:
raise NotImplementedError(
"model %s does not support updating records, mailgate API method message_update() is missing"
% model
)
if not len(res_ids):
new_res_id, attachment_ids = create_record(msg)
res_ids = [new_res_id]
# Store messages
context.update({"model": model})
if hasattr(model_pool, "history"):
model_pool.history(
cr,
uid,
res_ids,
_("receive"),
history=True,
subject=msg.get("subject"),
email=msg.get("to"),
details=msg.get("body"),
email_from=msg.get("from"),
email_cc=msg.get("cc"),
message_id=msg.get("message-id"),
references=msg.get("references", False) or msg.get("in-reply-to", False),
attach=attachments.items(),
email_date=msg.get("date"),
context=context,
)
else:
self.history(cr, uid, model, res_ids, msg, attachment_ids, context=context)
self.email_forward(cr, uid, model, res_ids, msg_txt)
return new_res_id
def parse_message(self, message, save_original=False):
"""Parses a string or email.message.Message representing an
RFC-2822 email, and returns a generic dict holding the
message details.
:param message: the message to parse
:type message: email.message.Message | string | unicode
:param bool save_original: whether the returned dict
should include an ``original`` entry with the base64
encoded source of the message.
:rtype: dict
:return: A dict with the following structure, where each
field may not be present if missing in original
message::
{ 'message-id': msg_id,
'subject': subject,
'from': from,
'to': to,
'cc': cc,
'headers' : { 'X-Mailer': mailer,
#.. all X- headers...
},
'subtype': msg_mime_subtype,
'body_text': plaintext_body
'body_html': html_body,
'attachments': [('file1', 'bytes'),
('file2', 'bytes') }
# ...
'original': source_of_email,
}
"""
msg_txt = message
if isinstance(message, str):
if message.startswith("undefinedFrom -"):
msg_txt = email.message_from_string(message.replace("undefinedFrom", "undefinedFromPop:", 1))
else:
msg_txt = email.message_from_string(message)
# Warning: message_from_string doesn't always work correctly on unicode,
# we must use utf-8 strings here :-(
if isinstance(message, unicode):
message = message.encode("utf-8")
msg_txt = email.message_from_string(message)
message_id = msg_txt.get("message-id", False)
msg = {}
if save_original:
# save original, we need to be able to read the original email sometimes
msg["original"] = message.as_string() if isinstance(message, Message) else message
msg["original"] = base64.b64encode(msg["original"]) # binary fields are b64
if not message_id:
# Very unusual situation, be we should be fault-tolerant here
message_id = str(time.time()) # KGB primax problem
msg_txt["message-id"] = message_id
_logger.info("Parsing Message without message-id, generating a random one: %s", message_id)
fields = msg_txt.keys()
msg["id"] = message_id
msg["message-id"] = message_id
if "Subject" in fields:
msg["subject"] = decode(msg_txt.get("Subject"))
if "Content-Type" in fields:
msg["content-type"] = msg_txt.get("Content-Type")
if "From" in fields:
msg["from"] = decode(msg_txt.get("From") or msg_txt.get_unixfrom())
if "To" in fields:
msg["to"] = decode(msg_txt.get("To"))
if "Delivered-To" in fields:
msg["to"] = decode(msg_txt.get("Delivered-To"))
if "CC" in fields:
msg["cc"] = decode(msg_txt.get("CC"))
if "Cc" in fields:
msg["cc"] = decode(msg_txt.get("Cc"))
if "Reply-To" in fields:
msg["reply"] = decode(msg_txt.get("Reply-To"))
if "Date" in fields:
try:
date_hdr = decode(msg_txt.get("Date"))
parsed_date = dateutil.parser.parse(date_hdr, fuzzy=True)
if parsed_date.utcoffset() is None:
# naive datetime, so we arbitrarily decide to make it
# UTC, there's no better choice. Should not happen,
# as RFC2822 requires timezone offset in Date headers.
stored_date = parsed_date.replace(tzinfo=pytz.utc)
else:
stored_date = parsed_date.astimezone(pytz.utc)
except Exception:
_logger.warning(
"Failed to parse Date header %r in incoming mail "
"with message-id %r, assuming current date/time.",
msg_txt.get("Date"),
message_id,
)
stored_date = datetime.datetime.now()
msg["date"] = stored_date.strftime("%Y-%m-%d %H:%M:%S")
if "Content-Transfer-Encoding" in fields:
msg["encoding"] = msg_txt.get("Content-Transfer-Encoding")
if "References" in fields:
msg["references"] = msg_txt.get("References")
if "In-Reply-To" in fields:
msg["in-reply-to"] = msg_txt.get("In-Reply-To")
msg["headers"] = {}
msg["subtype"] = "plain"
for item in msg_txt.items():
if item[0].startswith("X-"):
msg["headers"].update({item[0]: item[1]})
if not msg_txt.is_multipart() or "text/plain" in msg.get("content-type", ""):
encoding = msg_txt.get_content_charset()
body = tools.ustr(msg_txt.get_payload(decode=True), encoding, errors="replace")
if "text/html" in msg.get("content-type", ""):
msg["body_html"] = body
msg["subtype"] = "html"
body = tools.html2plaintext(body)
msg["body_text"] = tools.ustr(body, encoding, errors="replace")
attachments = []
if msg_txt.is_multipart() or "multipart/alternative" in msg.get("content-type", ""):
body = ""
if "multipart/alternative" in msg.get("content-type", ""):
msg["subtype"] = "alternative"
else:
msg["subtype"] = "mixed"
for part in msg_txt.walk():
if part.get_content_maintype() == "multipart":
continue
encoding = part.get_content_charset()
filename = part.get_filename()
if part.get_content_maintype() == "text":
content = part.get_payload(decode=True)
if filename:
attachments.append((filename, content))
content = tools.ustr(content, encoding, errors="replace")
if part.get_content_subtype() == "html":
msg["body_html"] = content
msg["subtype"] = "html" # html version prevails
body = tools.ustr(tools.html2plaintext(content))
body = body.replace(" ", "")
elif part.get_content_subtype() == "plain":
body = content
elif part.get_content_maintype() in ("application", "image"):
if filename:
attachments.append((filename, part.get_payload(decode=True)))
else:
res = part.get_payload(decode=True)
body += tools.ustr(res, encoding, errors="replace")
msg["body_text"] = body
msg["attachments"] = attachments
# for backwards compatibility:
msg["body"] = msg["body_text"]
msg["sub_type"] = msg["subtype"] or "plain"
return msg
def parse_message(self, message, save_original=False):
"""Parses a string or email.message.Message representing an
RFC-2822 email, and returns a generic dict holding the
message details.
:param message: the message to parse
:type message: email.message.Message | string | unicode
:param bool save_original: whether the returned dict
should include an ``original`` entry with the base64
encoded source of the message.
:rtype: dict
:return: A dict with the following structure, where each
field may not be present if missing in original
message::
{ 'message-id': msg_id,
'subject': subject,
'from': from,
'to': to,
'cc': cc,
'headers' : { 'X-Mailer': mailer,
#.. all X- headers...
},
'subtype': msg_mime_subtype,
'body_text': plaintext_body
'body_html': html_body,
'attachments': [('file1', 'bytes'),
('file2', 'bytes') }
# ...
'original': source_of_email,
}
"""
def unique_filename(attachments, file_name):
if attachments:
attachments_dict = dict(attachments)
if '.' in file_name:
name, extension = file_name.rsplit('.', 1)
else:
name = file_name
extension = ''
new_filename = name + '.' + extension
count = 0
while new_filename in attachments_dict:
count += 1
new_filename = "{name}-{count}.{extension}".format(name=name, count=count, extension=extension)
return new_filename
else:
return file_name
msg_txt = message
if isinstance(message, str):
msg_txt = email.message_from_string(message)
# Warning: message_from_string doesn't always work correctly on unicode,
# we must use utf-8 strings here :-(
if isinstance(message, unicode):
message = message.encode('utf-8')
msg_txt = email.message_from_string(message)
message_id = msg_txt.get('message-id', False)
msg = {}
if save_original:
# save original, we need to be able to read the original email sometimes
msg['original'] = message.as_string() if isinstance(message, Message) \
else message
msg['original'] = base64.b64encode(msg['original']) # binary fields are b64
if not message_id:
# Very unusual situation, be we should be fault-tolerant here
message_id = time.time()
msg_txt['message-id'] = message_id
_logger.info('Parsing Message without message-id, generating a random one: %s', message_id)
fields = msg_txt.keys()
msg['id'] = message_id
msg['message-id'] = message_id
if 'Subject' in fields:
msg['subject'] = decode(msg_txt.get('Subject'))
if 'Content-Type' in fields:
msg['content-type'] = msg_txt.get('Content-Type')
if 'From' in fields:
msg['from'] = decode(msg_txt.get('From') or msg_txt.get_unixfrom())
if 'To' in fields:
msg['to'] = decode(msg_txt.get('To'))
if 'Delivered-To' in fields:
msg['to'] = decode(msg_txt.get('Delivered-To'))
if 'CC' in fields:
msg['cc'] = decode(msg_txt.get('CC'))
if 'Cc' in fields:
msg['cc'] = decode(msg_txt.get('Cc'))
if 'Reply-To' in fields:
msg['reply'] = decode(msg_txt.get('Reply-To'))
if 'Date' in fields:
try:
date_hdr = decode(msg_txt.get('Date'))
parsed_date = dateutil.parser.parse(date_hdr, fuzzy=True)
if parsed_date.utcoffset() is None:
# naive datetime, so we arbitrarily decide to make it
# UTC, there's no better choice. Should not happen,
# as RFC2822 requires timezone offset in Date headers.
stored_date = parsed_date.replace(tzinfo=pytz.utc)
else:
stored_date = parsed_date.astimezone(pytz.utc)
except Exception:
_logger.warning('Failed to parse Date header %r in incoming mail '
'with message-id %r, assuming current date/time.',
msg_txt.get('Date'), message_id)
stored_date = datetime.datetime.now()
msg['date'] = stored_date.strftime("%Y-%m-%d %H:%M:%S")
if 'Content-Transfer-Encoding' in fields:
msg['encoding'] = msg_txt.get('Content-Transfer-Encoding')
if 'References' in fields:
msg['references'] = msg_txt.get('References')
if 'In-Reply-To' in fields:
msg['in-reply-to'] = msg_txt.get('In-Reply-To')
msg['headers'] = {}
msg['subtype'] = 'plain'
for item in msg_txt.items():
if item[0].startswith('X-'):
msg['headers'].update({item[0]: item[1]})
if not msg_txt.is_multipart() or 'text/plain' in msg.get('content-type', ''):
encoding = msg_txt.get_content_charset()
body = tools.ustr(msg_txt.get_payload(decode=True), encoding, errors='replace')
if 'text/html' in msg.get('content-type', ''):
msg['body_html'] = body
msg['subtype'] = 'html'
body = tools.html2plaintext(body)
msg['body_text'] = tools.ustr(body, encoding, errors='replace')
attachments = []
if msg_txt.is_multipart() or 'multipart/alternative' in msg.get('content-type', ''):
body = ""
if 'multipart/alternative' in msg.get('content-type', ''):
msg['subtype'] = 'alternative'
else:
msg['subtype'] = 'mixed'
for part in msg_txt.walk():
if part.get_content_maintype() == 'multipart':
continue
encoding = part.get_content_charset()
filename = part.get_filename()
if part.get_content_maintype() == 'text':
content = part.get_payload(decode=True)
if filename:
attachments.append((unique_filename(attachments, filename), content))
content = tools.ustr(content, encoding, errors='replace')
if part.get_content_subtype() == 'html':
msg['body_html'] = content
msg['subtype'] = 'html' # html version prevails
body = tools.ustr(tools.html2plaintext(content))
body = body.replace(' ', '')
elif part.get_content_subtype() == 'plain':
body = content
elif part.get_content_maintype() in ('application', 'image'):
if filename:
attachments.append((unique_filename(attachments, filename), part.get_payload(decode=True)))
else:
res = part.get_payload(decode=True)
body += tools.ustr(res, encoding, errors='replace')
msg['body_text'] = body
msg['attachments'] = attachments
# for backwards compatibility:
msg['body'] = msg['body_text']
msg['sub_type'] = msg['subtype'] or 'plain'
return msg
def Qatnf(psr, Query=('RaJ', 'DecJ')):
if psr == 'all': psr = ''
name = psr.split()
if len(name) == 2:
if name[0] == 'PSR':
psr = name[1]
try:
#for i in [1]:
#Try query the webserver
url = 'http://www.atnf.csiro.au/research/pulsar/psrcat/proc_form.php'
data = {}
if isinstance(Query, (basestring)):
Query = [Query]
for Q in Query:
data[Q] = Q
url_values = urllib.urlencode(data)
data = {
'ephemeris': 'short',
'startUserDefined': 'true',
'style': 'Publication quality',
'sort_attr': 'jname',
'sort_order': 'asc'
}
url_values += '&' + urllib.urlencode(data)
data = {'pulsar_names': psr}
url_values += '&' + urllib.urlencode(data)
data = {
'x_axis': '',
'x_scale': 'linear',
'y_axis': '',
'y_scale': 'linear',
'no_value': 'None',
'coords_unit': 'raj/decj',
'radius': '',
'coords_1': '',
'coords_2': '',
'fsize': '3'
}
url_values += '&' + urllib.urlencode(data)
data = {'state': 'query'}
url_values += '&' + urllib.urlencode(data)
data = {'table_bottom.x': '100', 'table_bottom.y': '100'}
url_values += '&' + urllib.urlencode(data)
full_url = url + '?' + url_values
order = []
for key in Query:
idx = url_values.find(key)
order.append((idx, key))
order.sort()
keylist = [x[1] for x in order]
the_page = urllib2.urlopen(full_url).read()
text = html2plaintext(the_page, encoding='UTF-8')
#print text
Nkey = len(keylist)
#print Nkey
#p = re.compile('(?P<Number>[ \t\n\r\f\v]\d+|\d+)(?P<line>(\s+(?P<value>-*\+*(\d{2,2}\:)*(\d+\.*\d*(e-*\+*\d+)*)|NONE|[JB]\d{4}[-\+]\d{2,4})(\s+(?P<uncertainty>\(\d{1,2}\))){0,1}(\s+(?P<ref>[a-z]+\+*\d{2}|\[\d+\]))*){1,%d})' % (Nkey), re.U)
p = re.compile(
'(?P<Number>\d+)(?P<line>(\s+(?P<value>-*\+*(\d{2,2}\:)*(\d+\.*\d*(e-*\+*\d+)*)|NONE|([JB]\d{4}[-\+]\d{2,4}))(\s+(?P<uncertainty>\(\d{1,2}\))){0,1}(\s+(?P<ref>[a-z]+\+*\d+|\[\d+\]))*){1,%d})'
% (Nkey), re.U)
pattern = re.compile(
'(?P<number>(?P<value>-*\+*(\d{2,2}\:)*\d+(\.\d+){0,1}(e-*\+*\d+){0,1}|NONE|[JB]\d{4}[-\+]\d{2,4})(\s+(?P<uncertainty>\(\d{1,2}\))){0,1}(\s+(?P<ref>[a-z]+\+{0,1}\d+|\[\d+\]))*)'
)
#finally:
if not psr == '':
m = p.search(text)
line = m.group('line')
#print 'line: \n',line
results = _retrieve(line, keylist, pattern)
return results
else:
results = []
for m in p.finditer(text):
line = m.group('line')
#print m.group()
#print 'line %s: \n%s' % (m.group('Number'),line)
results.append(_retrieve(line, keylist, pattern))
return results
except:
print 'web server query failed, try local database.\n'
return LQatnf(psr, Query)
def parse_message(self, message):
#TOCHECK: put this function in mailgateway module
if isinstance(message, unicode):
message = message.encode('utf-8')
msg_txt = email.message_from_string(message)
message_id = msg_txt.get('message-id', False)
msg = {}
fields = msg_txt.keys()
msg['id'] = message_id
msg['message-id'] = message_id
if 'Subject' in fields:
msg['subject'] = self._decode_header(msg_txt.get('Subject'))
if 'Content-Type' in fields:
msg['content-type'] = msg_txt.get('Content-Type')
if 'From' in fields:
msg['from'] = self._decode_header(msg_txt.get('From'))
if 'Delivered-To' in fields:
msg['to'] = self._decode_header(msg_txt.get('Delivered-To'))
if 'CC' in fields:
msg['cc'] = self._decode_header(msg_txt.get('CC'))
if 'Reply-to' in fields:
msg['reply'] = self._decode_header(msg_txt.get('Reply-To'))
if 'Date' in fields:
msg['date'] = self._decode_header(msg_txt.get('Date'))
if 'Content-Transfer-Encoding' in fields:
msg['encoding'] = msg_txt.get('Content-Transfer-Encoding')
if 'References' in fields:
msg['references'] = msg_txt.get('References')
if 'In-Reply-To' in fields:
msg['in-reply-to'] = msg_txt.get('In-Reply-To')
if 'X-Priority' in fields:
msg['priority'] = msg_txt.get('X-Priority', '3 (Normal)').split(' ')[0]
if not msg_txt.is_multipart() or 'text/plain' in msg.get('Content-Type', ''):
encoding = msg_txt.get_content_charset()
body = msg_txt.get_payload(decode=True)
msg['body'] = tools.ustr(body, encoding)
attachments = {}
has_plain_text = False
if msg_txt.is_multipart() or 'multipart/alternative' in msg.get('content-type', ''):
body = ""
for part in msg_txt.walk():
if part.get_content_maintype() == 'multipart':
continue
encoding = part.get_content_charset()
filename = part.get_filename()
if part.get_content_maintype()=='text':
content = part.get_payload(decode=True)
if filename:
attachments[filename] = content
elif not has_plain_text:
# main content parts should have 'text' maintype
# and no filename. we ignore the html part if
# there is already a plaintext part without filename,
# because presumably these are alternatives.
content = tools.ustr(content, encoding)
if part.get_content_subtype() == 'html':
body = tools.ustr(tools.html2plaintext(content))
elif part.get_content_subtype() == 'plain':
body = content
has_plain_text = True
elif part.get_content_maintype() in ('application', 'image'):
if filename :
attachments[filename] = part.get_payload(decode=True)
else:
res = part.get_payload(decode=True)
body += tools.ustr(res, encoding)
msg['body'] = body
msg['attachments'] = attachments
return msg
def parse_message(self, message, save_original=False):
"""Parses a string or email.message.Message representing an
RFC-2822 email, and returns a generic dict holding the
message details.
:param message: the message to parse
:type message: email.message.Message | string | unicode
:param bool save_original: whether the returned dict
should include an ``original`` entry with the base64
encoded source of the message.
:rtype: dict
:return: A dict with the following structure, where each
field may not be present if missing in original
message::
{ 'message-id': msg_id,
'subject': subject,
'from': from,
'to': to,
'cc': cc,
'headers' : { 'X-Mailer': mailer,
#.. all X- headers...
},
'subtype': msg_mime_subtype,
'body_text': plaintext_body
'body_html': html_body,
'attachments': [('file1', 'bytes'),
('file2', 'bytes') }
# ...
'original': source_of_email,
}
"""
msg_txt = message
if isinstance(message, str):
msg_txt = email.message_from_string(message)
# Warning: message_from_string doesn't always work correctly on unicode,
# we must use utf-8 strings here :-(
if isinstance(message, unicode):
message = message.encode('utf-8')
msg_txt = email.message_from_string(message)
message_id = msg_txt.get('message-id', False)
msg = {}
if save_original:
# save original, we need to be able to read the original email sometimes
msg['original'] = message.as_string() if isinstance(message, Message) \
else message
msg['original'] = base64.b64encode(msg['original']) # binary fields are b64
if not message_id:
# Very unusual situation, be we should be fault-tolerant here
message_id = time.time()
msg_txt['message-id'] = message_id
_logger.info('Parsing Message without message-id, generating a random one: %s', message_id)
fields = msg_txt.keys()
msg['id'] = message_id
msg['message-id'] = message_id
if 'Subject' in fields:
msg['subject'] = decode(msg_txt.get('Subject'))
if 'Content-Type' in fields:
msg['content-type'] = msg_txt.get('Content-Type')
if 'From' in fields:
msg['from'] = decode(msg_txt.get('From') or msg_txt.get_unixfrom())
if 'To' in fields:
msg['to'] = decode(msg_txt.get('To'))
if 'Delivered-To' in fields:
msg['to'] = decode(msg_txt.get('Delivered-To'))
if 'CC' in fields:
msg['cc'] = decode(msg_txt.get('CC'))
if 'Cc' in fields:
msg['cc'] = decode(msg_txt.get('Cc'))
if 'Reply-To' in fields:
msg['reply'] = decode(msg_txt.get('Reply-To'))
if 'Date' in fields:
date_hdr = decode(msg_txt.get('Date'))
msg['date'] = dateutil.parser.parse(date_hdr).strftime("%Y-%m-%d %H:%M:%S")
if 'Content-Transfer-Encoding' in fields:
msg['encoding'] = msg_txt.get('Content-Transfer-Encoding')
if 'References' in fields:
msg['references'] = msg_txt.get('References')
if 'In-Reply-To' in fields:
msg['in-reply-to'] = msg_txt.get('In-Reply-To')
msg['headers'] = {}
msg['subtype'] = 'plain'
for item in msg_txt.items():
if item[0].startswith('X-'):
msg['headers'].update({item[0]: item[1]})
if not msg_txt.is_multipart() or 'text/plain' in msg.get('content-type', ''):
encoding = msg_txt.get_content_charset()
body = msg_txt.get_payload(decode=True)
if 'text/html' in msg.get('content-type', ''):
msg['body_html'] = body
msg['subtype'] = 'html'
body = tools.html2plaintext(body)
msg['body_text'] = tools.ustr(body, encoding)
attachments = []
if msg_txt.is_multipart() or 'multipart/alternative' in msg.get('content-type', ''):
body = ""
if 'multipart/alternative' in msg.get('content-type', ''):
msg['subtype'] = 'alternative'
else:
msg['subtype'] = 'mixed'
for part in msg_txt.walk():
if part.get_content_maintype() == 'multipart':
continue
encoding = part.get_content_charset()
filename = part.get_filename()
if part.get_content_maintype()=='text':
content = part.get_payload(decode=True)
if filename:
attachments.append((filename, content))
content = tools.ustr(content, encoding)
if part.get_content_subtype() == 'html':
msg['body_html'] = content
msg['subtype'] = 'html' # html version prevails
body = tools.ustr(tools.html2plaintext(content))
elif part.get_content_subtype() == 'plain':
body = content
elif part.get_content_maintype() in ('application', 'image'):
if filename :
attachments.append((filename,part.get_payload(decode=True)))
else:
res = part.get_payload(decode=True)
body += tools.ustr(res, encoding)
msg['body_text'] = body
msg['attachments'] = attachments
# for backwards compatibility:
msg['body'] = msg['body_text']
msg['sub_type'] = msg['subtype'] or 'plain'
return msg
def Qatnf(psr, Query=('RaJ', 'DecJ'), condition=None):
if psr == 'all':psr=''
name = psr.split()
if len(name) == 2:
if name[0] == 'PSR':
psr = name[1]
#try:
#for i in [1]:
#Try query the webserver
url = 'http://www.atnf.csiro.au/research/pulsar/psrcat/proc_form.php'
data = {}
if isinstance(Query, (basestring)):
Query = [Query]
for Q in Query:
data[Q]=Q
url_values = urllib.urlencode(data)
#data = {'ephemeris':'short', 'startUserDefined':'true','style':'Publication quality','sort_attr':'jname','sort_order':'asc'}
#data = {'ephemeris':'short', 'startUserDefined':'true','style':'Long with last digit error','sort_attr':'jname','sort_order':'asc'}
data = {'ephemeris':'short', 'startUserDefined':'true','style':'Short without erros','sort_attr':'jname','sort_order':'asc'}
url_values += '&'+urllib.urlencode(data)
if condition:
data = {'pulsar_names':psr, 'condition':condition}
else:
data = {'pulsar_names':psr}
url_values += '&'+urllib.urlencode(data)
data = {'x_axis':'','x_scale':'linear','y_axis':'','y_scale':'linear',
'no_value':'None','coords_unit':'raj/decj','radius':'','coords_1':'',
'coords_2':'','fsize':'3' }
url_values += '&'+urllib.urlencode(data)
data = {'state':'query'}
url_values += '&'+urllib.urlencode(data)
data = {'table_bottom.x':'100', 'table_bottom.y':'100'}
url_values += '&'+urllib.urlencode(data)
full_url = url + '?' + url_values
order = []
for key in Query:
idx = url_values.find(key)
order.append((idx, key))
order.sort()
keylist = [x[1] for x in order]
the_page = urllib2.urlopen(full_url).read()
#print the_page
text = html2plaintext(the_page, encoding='UTF-8')
'''filer the text for artifacts [??]'''
artifacts = re.compile('(\[\d+\])', re.VERBOSE)
text = artifacts.sub('', text)
text = text.encode('ascii', 'ignore')
#print text
#print '\n'.join([x for x in text.split('\n') if not x == '' and not x.startswith(' http')][20:22])
#print '\n'.join(the_page.split('\n')[93:95])
#print '\n'.join([x for x in text.split('\n') if not x == '' and not x.startswith(' http')][22:-22])
text = '\n'.join([x for x in text.split('\n') if not x == '' and not x.startswith(' http')][22:-22])+'\n'
Nkey = len(keylist)
#print Nkey
#p = re.compile(r"""^(?P<Number>\d+)
#(?P<line>(\s+
#(?P<value>([-\+]{0,1}\d{2}\:\d{2}\:\d{2}\.\d+)|(\d+\.\d*(e-*\+*\d+)*)|NONE|\*|([JB]\d{4}[-\+]\d{2,4})|((AXP|HE|NRAD|XDINS|RRAT)(\,(AXP|HE|NRAD|XDINS|RRAT)){0,1}))
#(\s+(?P<uncertainty>\d{1,2})\s+){0,1}
#(?P<ref>\s*\[*[a-z]+\+{0,1}\d{2}\]*){0,1}
#){%d,%d})\s+\n""" % (Nkey,Nkey), re.VERBOSE)
p = re.compile(r"""(?P<Number>\d+)\s+
(?P<line>(.*))\n""" , re.VERBOSE)
pattern = re.compile(r"""
(?<=\s)(?P<value>((-*\+*(\d{2}\:){1,2}){0,1}\d+(\.\d*){0,1}([eE]-*\+*\d+){0,1})|NONE|\*|([JB]\d{4}[-\+]\d{2,5}([A-Z]|-\d|[a-z]{1,2}){0,1})|(AXP|HE|NRAD|XDINS|XINS|RRAT)(\,(AXP|HE|NRAD|XDINS|XINS|RRAT)){0,1})
((\s+)(?P<uncertainty>\d+)(?=\s)){0,1}
((\s*|\[)(?P<ref>(([a-z]{2,4}\+{0,1}\d{2})(\,\s+([a-z]+\+{0,1}\d{2}))*)|(?<=\s)\*)\]*){0,1}(\s%){0,1}
""", re.VERBOSE)
#(?<=[\s|\[])((?P<ref>(((\w{2,4}\+{0,1})(?=\d{2}))|\*))\]*){0,1}\s%{0,1}
#((\s+|\[)(?P<ref>(([a-z]{2,4}\+{0,1}(?=\d{2}))(\,\s+([a-z]+\+{0,1}\d{2}))*)|\*)\]*){0,1}\s%{0,1}
#((\s*|\[)(?P<ref>(([a-z]+\+{0,1}\d{2})(\,\s+([a-z]+\+{0,1}\d{2}))*)|(?<=\s)\*)\]*)(\s%){0,1}
#finally:
#print keylist
#print text
if not psr == '':
m = p.search(text)
line = ' '+m.group('line')
#print m.group('Number'),' :', line
results = _retrieve(line, keylist, pattern)
return results
else:
results = []
i = 0
for m in p.finditer(text):
try:
line = m.group('line')
info = _retrieve(line, keylist, pattern)
if not info == None:
results.append(info)
i+=1
except:
#print i
print 'line %s: \n%s' % (m.group('Number'),line)
return results
