def parse(self, content):
p = Parser()
msgobj = p.parsestr(content)
subject = self.parse_header_field(msgobj['Subject'])
attachments = []
body = []
html = []
self.parse_body(msgobj.walk(), attachments, body, html)
body = u'\n'.join(body)
html = u'\n'.join(html)
tos = self.get_address_list(msgobj, 'To')
tos.extend(self.get_address_list(msgobj, 'X-Original-To'))
ccs = self.get_address_list(msgobj, 'Cc')
resent_tos = self.get_address_list(msgobj, 'resent-to')
resent_ccs = self.get_address_list(msgobj, 'resent-cc')
from_field = parseaddr(msgobj.get('From'))
from_field = (self.parse_header_field(from_field[0]), from_field[1])
date = self.parse_date(msgobj.get("Date"))
return {
'msgobj': msgobj,
'date': date,
'subject': subject,
'body': body,
'html': html,
'from': from_field,
'to': tos,
'cc': ccs,
'resent_to': resent_tos,
'resent_cc': resent_ccs,
'attachments': attachments
}
def poll_newsgroup(mlist, conn, first, last, glock):
listname = mlist.internal_name()
# NEWNEWS is not portable and has synchronization issues.
for num in range(first, last):
glock.refresh()
try:
headers = conn.head(repr(num))[3]
found_to = False
beenthere = False
for header in headers:
i = header.find(':')
value = header[:i].lower()
if i > 0 and value == 'to':
found_to = True
# FIXME 2010-02-16 barry use List-Post header.
if value <> 'x-beenthere':
continue
if header[i:] == ': %s' % mlist.posting_address:
beenthere = True
break
if not beenthere:
body = conn.body(repr(num))[3]
# Usenet originated messages will not have a Unix envelope
# (i.e. "From " header). This breaks Pipermail archiving, so
# we will synthesize one. Be sure to use the format searched
# for by mailbox.UnixMailbox._isrealfromline(). BAW: We use
# the -bounces address here in case any downstream clients use
# the envelope sender for bounces; I'm not sure about this,
# but it's the closest to the old semantics.
lines = ['From %s %s' % (mlist.GetBouncesEmail(),
time.ctime(time.time()))]
lines.extend(headers)
lines.append('')
lines.extend(body)
lines.append('')
p = Parser(Message.Message)
try:
msg = p.parsestr(NL.join(lines))
except email.Errors.MessageError as e:
log.error('email package exception for %s:%d\n%s',
mlist.linked_newsgroup, num, e)
raise _ContinueLoop
if found_to:
del msg['X-Originally-To']
msg['X-Originally-To'] = msg['To']
del msg['To']
msg['To'] = mlist.posting_address
# Post the message to the locked list
inq = Switchboard(config.INQUEUE_DIR)
inq.enqueue(msg,
listid=mlist.list_id,
fromusenet=True)
log.info('posted to list %s: %7d', listname, num)
except nntplib.NNTPError as e:
log.exception('NNTP error for list %s: %7d', listname, num)
except _ContinueLoop:
continue
# Even if we don't post the message because it was seen on the
# list already, update the watermark
mlist.usenet_watermark = num
def poll_newsgroup(mlist, conn, first, last, glock):
listname = mlist.internal_name()
# NEWNEWS is not portable and has synchronization issues.
for num in range(first, last):
glock.refresh()
try:
headers = conn.head(repr(num))[3]
found_to = False
beenthere = False
for header in headers:
i = header.find(':')
value = header[:i].lower()
if i > 0 and value == 'to':
found_to = True
# FIXME 2010-02-16 barry use List-Post header.
if value <> 'x-beenthere':
continue
if header[i:] == ': %s' % mlist.posting_address:
beenthere = True
break
if not beenthere:
body = conn.body(repr(num))[3]
# Usenet originated messages will not have a Unix envelope
# (i.e. "From " header). This breaks Pipermail archiving, so
# we will synthesize one. Be sure to use the format searched
# for by mailbox.UnixMailbox._isrealfromline(). BAW: We use
# the -bounces address here in case any downstream clients use
# the envelope sender for bounces; I'm not sure about this,
# but it's the closest to the old semantics.
lines = [
'From %s %s' %
(mlist.GetBouncesEmail(), time.ctime(time.time()))
]
lines.extend(headers)
lines.append('')
lines.extend(body)
lines.append('')
p = Parser(Message.Message)
try:
msg = p.parsestr(NL.join(lines))
except email.Errors.MessageError as e:
log.error('email package exception for %s:%d\n%s',
mlist.linked_newsgroup, num, e)
raise _ContinueLoop
if found_to:
del msg['X-Originally-To']
msg['X-Originally-To'] = msg['To']
del msg['To']
msg['To'] = mlist.posting_address
# Post the message to the locked list
inq = Switchboard(config.INQUEUE_DIR)
inq.enqueue(msg, listid=mlist.list_id, fromusenet=True)
log.info('posted to list %s: %7d', listname, num)
except nntplib.NNTPError as e:
log.exception('NNTP error for list %s: %7d', listname, num)
except _ContinueLoop:
continue
# Even if we don't post the message because it was seen on the
# list already, update the watermark
mlist.usenet_watermark = num
def parse(self, content):
p = Parser()
msgobj = p.parsestr(content)
subject = self.parse_header_field(msgobj['Subject'])
attachments = []
body = []
html = []
self.parse_body(msgobj.walk(), attachments, body, html)
body = u'\n'.join(body)
html = u'\n'.join(html)
tos = self.get_address_list(msgobj, 'To')
tos.extend(self.get_address_list(msgobj, 'X-Original-To'))
ccs = self.get_address_list(msgobj, 'Cc')
resent_tos = self.get_address_list(msgobj, 'resent-to')
resent_ccs = self.get_address_list(msgobj, 'resent-cc')
from_field = parseaddr(msgobj.get('From'))
from_field = (self.parse_header_field(from_field[0]), from_field[1])
date = self.parse_date(msgobj.get("Date"))
return {
'msgobj': msgobj,
'date': date,
'subject': subject,
'body': body,
'html': html,
'from': from_field,
'to': tos,
'cc': ccs,
'resent_to': resent_tos,
'resent_cc': resent_ccs,
'attachments': attachments
}
def parse(content):
p = EmailParser()
msgobj = p.parsestr(content)
if msgobj['Subject'] is not None:
decodefrag = decode_header(msgobj['Subject'])
subj_fragments = []
for s, enc in decodefrag:
if enc:
s = unicode(s , enc).encode('utf8', 'replace')
subj_fragments.append(s)
subject = ''.join(subj_fragments)
else:
subject = None
attachments = []
body = None
html = None
images = []
images_content_type = [
"image/jpg",
"image/jpeg",
"image/png",
"image/tiff"
"application/pdf",
]
for part in msgobj.walk():
print part.get_content_type()
attachment = parse_attachment(part)
if attachment:
attachments.append(attachment)
elif part.get_content_type() == "text/plain":
if body is None:
body = ""
body += unicode(
part.get_payload(decode=True),
part.get_content_charset(),
'replace'
).encode('utf8', 'replace')
elif part.get_content_type() == "text/html":
if html is None:
html = ""
html += unicode(
part.get_payload(decode=True),
part.get_content_charset(),
'replace'
).encode('utf8', 'replace')
elif part.get_content_type() in images_content_type:
images.append(StringIO(part.get_payload(decode=True)))
return {
'subject': subject,
'body': body,
'html': html,
'from': parseaddr(msgobj.get('From'))[1],
'to': parseaddr(msgobj.get('To'))[1],
'attachments': attachments,
'images': images,
}
class lector_email:
"""Clase utilizada para leer los emails de comandos"""
def __init__(self, usuario, clave):
"""Se establece conexion con el servidor pop de gmail"""
self.usuario = usuario
self.clave = clave
self.conectar()
self.codigos = []
self.codigos.append("ok") # Definimos un codigo inicial y basico
def conectar(self):
"""Se conecta con el servidor pop3"""
self.m = poplib.POP3_SSL("pop.gmail.com", 995)
self.m.user(self.usuario)
self.m.pass_(self.clave)
print "conectado al server pop3"
def definir_mensaje(self, nuevo_mensaje):
"""Agrega nuevo_mensaje a la lista sef.codigo"""
self.codigos.append(nuevo_mensaje.lower())
def cerrar_conexion(self):
"""Encargado de cerrar la coneccion"""
self.m.quit()
def actualizar(self):
"""Ontiene los datos del servidor"""
print "llamada a actualizar"
# Se obtiene el numero de mensajes pendientes y se hace un
# bucle para cada mensaje
self.numero = len(self.m.list()[1])
for i in range(self.numero):
# print "Mensaje numero"+str(i+1)
# print "--------------------"
# Se lee el mensaje
self.response, self.headerLines, self.bytes = self.m.retr(i + 1)
# Se mete todo el mensaje en un unico string
self.mensaje = "\n".join(self.headerLines)
# Se parsea el mensaje
self.p = Parser()
self.email = self.p.parsestr(self.mensaje)
# Se sacan por pantalla los campos from, to y subject
print "From: " + self.email["From"]
print "To: " + self.email["To"]
print "Subject: " + self.email["Subject"]
print self.email.get_content_type()
print self.email.get_payload(decode=True)
print self.email.is_multipart()
self.tipo = self.email.get_content_type()
for codigo in self.codigos:
if ("text/plain" == self.tipo) and (self.email.get_payload(decode=True).lower() == codigo):
# Si es texto plano, se escribe en pantalla
print self.email.get_payload(decode=True)
self.m.dele(i + 1) # le sumamos 1 ya que python cuenta desde 0 y los emails desde 1
self.cerrar_conexion() # nos desconectamos solo si borramos algo, y lo borramos solo si es un mensaje de "ok"
self.conectar() # y de esta forma se guardan los archivos en el server
return codigo # retorna el comando que se encontro en el e-mail
return False # retorna falso en caso de no encontrar nada
def parse(content):
p = EmailParser()
if type(content) == str:
msgobj = p.parsestr(content)
else:
msgobj = p.parse(content)
attachments = []
return _parse2(msgobj, attachments)
def parse(content):
p = EmailParser()
if type(content) == str:
msgobj = p.parsestr(content)
else:
msgobj = p.parse(content)
attachments = []
return _parse2(msgobj, attachments)
def parse(content):
"""
parse email
"""
p = EmailParser()
#check content is a file or text
#if content is path...
#msgobj = p.parse(content)
msgobj = p.parsestr(content)
if msgobj['Subject'] is not None:
decodefrag = decode_header(msgobj['Subject'])
subj_fragments = []
for s , enc in decodefrag:
if enc:
s = unicode(s , enc).encode('utf8','replace')
subj_fragments.append(s)
subject = ''.join(subj_fragments)
else:
subject = None
attachments = []
body = None
html = None
for part in msgobj.walk():
attachment = parse_attachment(part)
if attachment:
attachments.append(attachment)
elif part.get_content_type() == "text/plain":
if body is None:
body = ""
if part.get_content_charset:
body += part.get_payload(decode=True)
else:
body += unicode(
part.get_payload(decode=True),
part.get_content_charset(),
'replace'
).encode('utf8','replace')
elif part.get_content_type() == "text/html":
if html is None:
html = ""
html += unicode(
part.get_payload(decode=True),
part.get_content_charset(),
'replace'
).encode('utf8','replace')
return {
'subject' : subject,
'body' : body,
'html' : html,
'from' : parseaddr(msgobj.get('From'))[1],
'to' : parseaddr(msgobj.get('To'))[1],
'date' : parse_date(msgobj.get('Date')),
'attachments': attachments,
}
def _checkModeratorMessage(self, messageText, sender, moderator, group, postingText):
p = Parser()
msg = p.parsestr(messageText)
headers = dict(msg.items())
del headers['Message-ID']
self.assertEqual(
headers,
{'From': sender,
'To': moderator,
'Subject': 'Moderate new %s message: activities etc' % (group,),
'Content-Type': 'message/rfc822'})
posting = p.parsestr(postingText)
attachment = msg.get_payload()[0]
for header in ['from', 'to', 'subject', 'message-id', 'newsgroups']:
self.assertEqual(posting[header], attachment[header])
self.assertEqual(posting.get_payload(), attachment.get_payload())
def _checkModeratorMessage(self, messageText, sender, moderator, group, postingText):
p = Parser()
msg = p.parsestr(messageText)
headers = dict(msg.items())
del headers['Message-ID']
self.assertEquals(
headers,
{'From': sender,
'To': moderator,
'Subject': 'Moderate new %s message: activities etc' % (group,),
'Content-Type': 'message/rfc822'})
posting = p.parsestr(postingText)
attachment = msg.get_payload()[0]
for header in ['from', 'to', 'subject', 'message-id', 'newsgroups']:
self.assertEquals(posting[header], attachment[header])
self.assertEquals(posting.get_payload(), attachment.get_payload())
def parse(self, content):
p = Parser()
msgobj = p.parsestr(content)
if msgobj['Subject'] is not None:
decodefrag = decode_header(msgobj['Subject'])
subj_fragments = []
for s , enc in decodefrag:
if enc:
s = unicode(s , enc).encode('utf8','replace')
subj_fragments.append(s)
subject = ''.join(subj_fragments)
else:
subject = None
subject = subject.replace('\n\t', " ")
attachments = []
body = None
html = None
for part in msgobj.walk():
attachment = self.parse_attachment(part)
if attachment:
attachments.append(attachment)
elif part.get_content_type() == "text/plain":
if body is None:
body = ""
charset = part.get_content_charset() or 'ascii'
body += unicode(
part.get_payload(decode=True),
charset,
'replace').encode('utf8','replace')
elif part.get_content_type() == "text/html":
if html is None:
html = ""
charset = part.get_content_charset() or 'ascii'
html += unicode(
part.get_payload(decode=True),
charset,
'replace').encode('utf8','replace')
tos = getaddresses(msgobj.get_all('To', []))
ccs = getaddresses(msgobj.get_all('Cc', []))
resent_tos = getaddresses(msgobj.get_all('resent-to', []))
resent_ccs = getaddresses(msgobj.get_all('resent-cc', []))
date = self.parse_date(msgobj.get("Date"))
return {
'msgobj': msgobj,
'date': date,
'subject': subject,
'body': body,
'html': html,
'from': parseaddr(msgobj.get('From')),
'to': tos,
'cc': ccs,
'resent_to': resent_tos,
'resent_cc': resent_ccs,
'attachments': attachments
}
def stepCheckBugMessageNotificationReAssign(self, sequence=None, sequence_list=None, **kw):
"""
Check the bug message when re-assign
"""
last_message = self.portal.MailHost._last_message
self.assertNotEquals((), last_message)
mfrom, mto, messageText = last_message
from email.Parser import Parser
p = Parser()
m = p.parsestr(messageText)
self.assertTrue('Re-assign!' in m.get_payload()[0].get_payload(decode=True))
def check_email(self, ip, username, password, hash_code, threshold):
self.log.info("checking inbox for matching hash code %s" % hash_code)
subject_to_match = self.generate_subject(hash_code)
start = time.time()
found = False
parser = Parser()
already_slept = False
# continue to loop over the inbox messages while we haven't found
# what we're looking for and we're still under our time threshold
self.log.info("starting inbox-checking loop")
while not found and time.time() - start < threshold:
self.log.info("checking inbox")
# the only way to fail the check from a test case is to check the inbox
# immediately, so we have to pass that option in manually
if self.options.get("nofirstsleep") and not already_slept: already_slept = True
else: time.sleep(int(self.config.get("Inbox_plugin", "check_delay")))
# we re-connect every time because for some reason sometimes the
# pop list gets cached, so re-listing it doesn't do any good.
self.log.info("connecting and logging in")
inbox = self.default_class(ip, self.port)
inbox.getwelcome()
inbox.user(username)
inbox.pass_(password)
self.log.info("getting inbox status")
inbox.stat()
status, messages, octets = inbox.list()
# loop over the messages
self.log.info("looping over %d messages", len(messages))
for message in messages:
index, size = message.split(" ")
status, message, size = inbox.retr(index)
message = "\n".join(message)
# try to match the email subject to the subject we're expecting
headers = parser.parsestr(message, True)
if headers["Subject"] == subject_to_match:
self.log.info("found match")
inbox.dele(index)
found = True
break
self.log.info("logging out of inbox")
inbox.quit()
return found
def _checkModeratorMessage(self, messageText, sender, moderator, group, postingText):
p = Parser()
msg = p.parsestr(messageText)
headers = dict(msg.items())
del headers["Message-ID"]
self.assertEqual(
headers,
{
"From": sender,
"To": moderator,
"Subject": "Moderate new %s message: activities etc" % (group,),
"Content-Type": "message/rfc822",
},
)
posting = p.parsestr(postingText)
attachment = msg.get_payload()[0]
for header in ["from", "to", "subject", "message-id", "newsgroups"]:
self.assertEqual(posting[header], attachment[header])
self.assertEqual(posting.get_payload(), attachment.get_payload())
def parse(content):
"""
Parse the email and return a dictionary of relevant data.
"""
p = EmailParser()
msgobj = p.parsestr(content)
if msgobj['Subject'] is not None:
decodefrag = decode_header(msgobj['Subject'])
subj_fragments = []
for s , enc in decodefrag:
if enc:
s = unicode(s , enc).encode('utf8','replace')
subj_fragments.append(s)
subject = ''.join(subj_fragments)
else:
subject = None
attachments = []
body = None
html = None
for part in msgobj.walk():
attachment = parse_attachment(part)
if attachment:
attachments.append(attachment)
elif part.get_content_type() == "text/plain":
if body is None:
body = ""
body += unicode(
part.get_payload(decode=True),
part.get_content_charset(),
'replace'
).encode('utf8','replace')
elif part.get_content_type() == "text/html":
if html is None:
html = ""
html += unicode(
part.get_payload(decode=True),
part.get_content_charset(),
'replace'
).encode('utf8','replace')
return {
'subject' : subject,
'body' : body,
'html' : html,
'from' : parseaddr(msgobj.get('From'))[1], # Leave off the name and only return the address
'to' : parseaddr(msgobj.get('To'))[1], # Leave off the name and only return the address
'attachments': attachments,
}
def parse(content):
"""
Parse the email and return a dictionary of relevant data.
"""
p = EmailParser()
msgobj = p.parsestr(content)
if msgobj['Subject'] is not None:
decodefrag = decode_header(msgobj['Subject'])
subj_fragments = []
for s, enc in decodefrag:
if enc:
s = unicode(s, enc).encode('utf8', 'replace')
subj_fragments.append(s)
subject = ''.join(subj_fragments)
else:
subject = None
attachments = []
body = None
html = None
for part in msgobj.walk():
attachment = parse_attachment(part)
if attachment:
attachments.append(attachment)
elif part.get_content_type() == "text/plain":
if body is None:
body = ""
body += unicode(part.get_payload(decode=True),
part.get_content_charset(),
'replace').encode('utf8', 'replace')
elif part.get_content_type() == "text/html":
if html is None:
html = ""
html += unicode(part.get_payload(decode=True),
part.get_content_charset(),
'replace').encode('utf8', 'replace')
return {
'subject': subject,
'body': body,
'html': html,
'from': parseaddr(msgobj.get(
'From'))[1], # Leave off the name and only return the address
'to': parseaddr(msgobj.get('To'))
[1], # Leave off the name and only return the address
'attachments': attachments,
}
class PopBox:
def __init__(self, user, password, host, port = 110, ssl = False):
self.user = user
self.password = password
self.host = host
self.port = int(port)
self.ssl = ssl
self.mbox = None
self.parser = EmailParser()
def __connect(self):
if not self.ssl:
self.mbox = poplib.POP3(self.host, self.port)
else:
self.mbox = poplib.POP3_SSL(self.host, self.port)
self.mbox.user(self.user)
self.mbox.pass_(self.password)
def get_mails(self):
self.__connect()
messages = []
print "Starting reading POP messages"
msgs = self.mbox.list()[1]
print "POP messages readed: %i" % (len(msgs))
for msg in msgs:
msgNum = int(msg.split(" ")[0])
msgSize = int(msg.split(" ")[1])
# retrieve only the header
st = "\n".join(self.mbox.top(msgNum, 0)[1])
print st
print "----------------------------------------"
msg = self.parser.parsestr(st, True) # header only
sub = utils.mime_decode(msg.get("Subject"))
msgid = msg.get("Message-Id")
if not msgid:
msgid = hash(msg.get("Received") + sub)
fr = utils.mime_decode(msg.get("From"))
messages.append( [msgid, sub, fr] )
self.mbox.quit()
return messages
def parse(content):
"""Parse the content string of an email message and return a dict of the parts"""
p = EmailParser()
msgobj = p.parsestr(content)
msgheaders = {}
for header_key in header_parts:
header_val = get_header_component (msgobj, header_key)
if header_val and header_val != 'None':
msgheaders[header_key] = header_val
attachments = []
body = None
html = None
for part in msgobj.walk():
attachment = parse_attachment(part)
if attachment:
attachments.append(attachment)
elif part.get_content_type() == "text/plain":
if body is None:
body = ""
body += unicode(
part.get_payload(decode=True),
unicode_encoding if part.get_content_charset() is None else part.get_content_charset(),
'replace'
).encode(unicode_encoding,'replace')
elif part.get_content_type() == "text/html":
if html is None:
html = ""
html += unicode(
part.get_payload(decode=True),
unicode_encoding if part.get_content_charset() is None else part.get_content_charset(),
'replace'
).encode(unicode_encoding,'replace')
return {
'headers' : msgheaders,
'body' : body,
'html' : html,
'attachments': attachments,
}
def parse(content):
"""Parse the content string of an email message and return a dict of the parts"""
p = EmailParser()
msgobj = p.parsestr(content)
msgheaders = {}
for header_key in header_parts:
header_val = get_header_component(msgobj, header_key)
if header_val and header_val != 'None':
msgheaders[header_key] = header_val
attachments = []
body = None
html = None
for part in msgobj.walk():
attachment = parse_attachment(part)
if attachment:
attachments.append(attachment)
elif part.get_content_type() == "text/plain":
if body is None:
body = ""
body += unicode(
part.get_payload(decode=True),
unicode_encoding if part.get_content_charset() is None else
part.get_content_charset(),
'replace').encode(unicode_encoding, 'replace')
elif part.get_content_type() == "text/html":
if html is None:
html = ""
html += unicode(
part.get_payload(decode=True),
unicode_encoding if part.get_content_charset() is None else
part.get_content_charset(),
'replace').encode(unicode_encoding, 'replace')
return {
'headers': msgheaders,
'body': body,
'html': html,
'attachments': attachments,
}
def decryptSmime(mlist, msg, msgdata):
"""Returns (encrypted (bool), signed (bool)), msg is replaced with
decrypted msg"""
# FIXME this implementation is _very_ crude.
# merge some stuff with decryptGpg
encrypted = False
signed = False
plaintext = None
ciphertext = None
if msg.get_content_type()=="application/x-pkcs7-mime":
sm = SMIMEUtils.SMIMEHelper(mlist)
ciphertext = msg.as_string()
(plaintext, signed) = sm.decryptMessage(ciphertext)
else:
# don't touch the message if it's no S/MIME
return (encrypted, signed)
parser = Parser()
tmpmsg = parser.parsestr(plaintext)
msg.del_param("x-action")
for i in ('Content-Type','Content-Disposition','Content-Transfer-Encoding'):
if tmpmsg.has_key(i):
if msg.has_key(i):
msg.replace_header(i,tmpmsg.get(i))
else:
msg.add_header(i,tmpmsg.get(i))
tmppayload = tmpmsg.get_payload()
msg.set_payload(tmppayload)
if encrypted:
msg.add_header('X-Mailman-SLS-decrypted', 'Yes')
return (encrypted, signed)
syslog('gpg','Unable to decrypt GPG dataMod')
raise Errors.RejectMessage, "Unable to decrypt mail!Mod"
else:
encrypted = True
if key_ids:
signed = True
if not encrypted:
return (encrypted, signed, key_ids)
# Check decryption result
# Check transfer type
parser = Parser()
tmpmsg = parser.parsestr(plaintext)
if msg.get_content_type()=='application/pgp':
msg.set_type("text/plain")
msg.del_param("x-action")
for i in ('Content-Type','Content-Disposition','Content-Transfer-Encoding'):
if tmpmsg.has_key(i):
if msg.has_key(i):
msg.replace_header(i,tmpmsg.get(i))
else:
msg.add_header(i,tmpmsg.get(i))
if tmpmsg.is_multipart():
msg.set_payload(None)
for i in tmpmsg.get_payload():
msg.attach(i)
else:
tmppayload = tmpmsg.get_payload()
def close(self):
p = Parser()
return p.parsestr(''.join(self.content))
class gnuarch_source(converter_source, commandline):
class gnuarch_rev(object):
def __init__(self, rev):
self.rev = rev
self.summary = ''
self.date = None
self.author = ''
self.continuationof = None
self.add_files = []
self.mod_files = []
self.del_files = []
self.ren_files = {}
self.ren_dirs = {}
def __init__(self, ui, path, rev=None):
super(gnuarch_source, self).__init__(ui, path, rev=rev)
if not os.path.exists(os.path.join(path, '{arch}')):
raise NoRepo(
_("%s does not look like a GNU Arch repository") % path)
# Could use checktool, but we want to check for baz or tla.
self.execmd = None
if util.find_exe('baz'):
self.execmd = 'baz'
else:
if util.find_exe('tla'):
self.execmd = 'tla'
else:
raise util.Abort(_('cannot find a GNU Arch tool'))
commandline.__init__(self, ui, self.execmd)
self.path = os.path.realpath(path)
self.tmppath = None
self.treeversion = None
self.lastrev = None
self.changes = {}
self.parents = {}
self.tags = {}
self.modecache = {}
self.catlogparser = Parser()
self.locale = locale.getpreferredencoding()
self.archives = []
def before(self):
# Get registered archives
self.archives = [
i.rstrip('\n') for i in self.runlines0('archives', '-n')
]
if self.execmd == 'tla':
output = self.run0('tree-version', self.path)
else:
output = self.run0('tree-version', '-d', self.path)
self.treeversion = output.strip()
# Get name of temporary directory
version = self.treeversion.split('/')
self.tmppath = os.path.join(tempfile.gettempdir(),
'hg-%s' % version[1])
# Generate parents dictionary
self.parents[None] = []
treeversion = self.treeversion
child = None
while treeversion:
self.ui.status(_('analyzing tree version %s...\n') % treeversion)
archive = treeversion.split('/')[0]
if archive not in self.archives:
self.ui.status(
_('tree analysis stopped because it points to '
'an unregistered archive %s...\n') % archive)
break
# Get the complete list of revisions for that tree version
output, status = self.runlines('revisions', '-r', '-f',
treeversion)
self.checkexit(status,
'failed retrieveing revisions for %s' % treeversion)
# No new iteration unless a revision has a continuation-of header
treeversion = None
for l in output:
rev = l.strip()
self.changes[rev] = self.gnuarch_rev(rev)
self.parents[rev] = []
# Read author, date and summary
catlog, status = self.run('cat-log', '-d', self.path, rev)
if status:
catlog = self.run0('cat-archive-log', rev)
self._parsecatlog(catlog, rev)
# Populate the parents map
self.parents[child].append(rev)
# Keep track of the current revision as the child of the next
# revision scanned
child = rev
# Check if we have to follow the usual incremental history
# or if we have to 'jump' to a different treeversion given
# by the continuation-of header.
if self.changes[rev].continuationof:
treeversion = '--'.join(
self.changes[rev].continuationof.split('--')[:-1])
break
# If we reached a base-0 revision w/o any continuation-of
# header, it means the tree history ends here.
if rev[-6:] == 'base-0':
break
def after(self):
self.ui.debug('cleaning up %s\n' % self.tmppath)
shutil.rmtree(self.tmppath, ignore_errors=True)
def getheads(self):
return self.parents[None]
def getfile(self, name, rev):
if rev != self.lastrev:
raise util.Abort(_('internal calling inconsistency'))
# Raise IOError if necessary (i.e. deleted files).
if not os.path.exists(os.path.join(self.tmppath, name)):
raise IOError
data, mode = self._getfile(name, rev)
self.modecache[(name, rev)] = mode
return data
def getmode(self, name, rev):
return self.modecache[(name, rev)]
def getchanges(self, rev):
self.modecache = {}
self._update(rev)
changes = []
copies = {}
for f in self.changes[rev].add_files:
changes.append((f, rev))
for f in self.changes[rev].mod_files:
changes.append((f, rev))
for f in self.changes[rev].del_files:
changes.append((f, rev))
for src in self.changes[rev].ren_files:
to = self.changes[rev].ren_files[src]
changes.append((src, rev))
changes.append((to, rev))
copies[to] = src
for src in self.changes[rev].ren_dirs:
to = self.changes[rev].ren_dirs[src]
chgs, cps = self._rendirchanges(src, to)
changes += [(f, rev) for f in chgs]
copies.update(cps)
self.lastrev = rev
return sorted(set(changes)), copies
def getcommit(self, rev):
changes = self.changes[rev]
return commit(author=changes.author,
date=changes.date,
desc=changes.summary,
parents=self.parents[rev],
rev=rev)
def gettags(self):
return self.tags
def _execute(self, cmd, *args, **kwargs):
cmdline = [self.execmd, cmd]
cmdline += args
cmdline = [util.shellquote(arg) for arg in cmdline]
cmdline += ['>', util.nulldev, '2>', util.nulldev]
cmdline = util.quotecommand(' '.join(cmdline))
self.ui.debug(cmdline, '\n')
return os.system(cmdline)
def _update(self, rev):
self.ui.debug('applying revision %s...\n' % rev)
changeset, status = self.runlines('replay', '-d', self.tmppath, rev)
if status:
# Something went wrong while merging (baz or tla
# issue?), get latest revision and try from there
shutil.rmtree(self.tmppath, ignore_errors=True)
self._obtainrevision(rev)
else:
old_rev = self.parents[rev][0]
self.ui.debug('computing changeset between %s and %s...\n' %
(old_rev, rev))
self._parsechangeset(changeset, rev)
def _getfile(self, name, rev):
mode = os.lstat(os.path.join(self.tmppath, name)).st_mode
if stat.S_ISLNK(mode):
data = os.readlink(os.path.join(self.tmppath, name))
mode = mode and 'l' or ''
else:
data = open(os.path.join(self.tmppath, name), 'rb').read()
mode = (mode & 0111) and 'x' or ''
return data, mode
def _exclude(self, name):
exclude = ['{arch}', '.arch-ids', '.arch-inventory']
for exc in exclude:
if name.find(exc) != -1:
return True
return False
def _readcontents(self, path):
files = []
contents = os.listdir(path)
while len(contents) > 0:
c = contents.pop()
p = os.path.join(path, c)
# os.walk could be used, but here we avoid internal GNU
# Arch files and directories, thus saving a lot time.
if not self._exclude(p):
if os.path.isdir(p):
contents += [os.path.join(c, f) for f in os.listdir(p)]
else:
files.append(c)
return files
def _rendirchanges(self, src, dest):
changes = []
copies = {}
files = self._readcontents(os.path.join(self.tmppath, dest))
for f in files:
s = os.path.join(src, f)
d = os.path.join(dest, f)
changes.append(s)
changes.append(d)
copies[d] = s
return changes, copies
def _obtainrevision(self, rev):
self.ui.debug('obtaining revision %s...\n' % rev)
output = self._execute('get', rev, self.tmppath)
self.checkexit(output)
self.ui.debug('analyzing revision %s...\n' % rev)
files = self._readcontents(self.tmppath)
self.changes[rev].add_files += files
def _stripbasepath(self, path):
if path.startswith('./'):
return path[2:]
return path
def _parsecatlog(self, data, rev):
try:
catlog = self.catlogparser.parsestr(data)
# Commit date
self.changes[rev].date = util.datestr(
util.strdate(catlog['Standard-date'], '%Y-%m-%d %H:%M:%S'))
# Commit author
self.changes[rev].author = self.recode(catlog['Creator'])
# Commit description
self.changes[rev].summary = '\n\n'.join(
(catlog['Summary'], catlog.get_payload()))
self.changes[rev].summary = self.recode(self.changes[rev].summary)
# Commit revision origin when dealing with a branch or tag
if 'Continuation-of' in catlog:
self.changes[rev].continuationof = self.recode(
catlog['Continuation-of'])
except Exception:
raise util.Abort(_('could not parse cat-log of %s') % rev)
def _parsechangeset(self, data, rev):
for l in data:
l = l.strip()
# Added file (ignore added directory)
if l.startswith('A') and not l.startswith('A/'):
file = self._stripbasepath(l[1:].strip())
if not self._exclude(file):
self.changes[rev].add_files.append(file)
# Deleted file (ignore deleted directory)
elif l.startswith('D') and not l.startswith('D/'):
file = self._stripbasepath(l[1:].strip())
if not self._exclude(file):
self.changes[rev].del_files.append(file)
# Modified binary file
elif l.startswith('Mb'):
file = self._stripbasepath(l[2:].strip())
if not self._exclude(file):
self.changes[rev].mod_files.append(file)
# Modified link
elif l.startswith('M->'):
file = self._stripbasepath(l[3:].strip())
if not self._exclude(file):
self.changes[rev].mod_files.append(file)
# Modified file
elif l.startswith('M'):
file = self._stripbasepath(l[1:].strip())
if not self._exclude(file):
self.changes[rev].mod_files.append(file)
# Renamed file (or link)
elif l.startswith('=>'):
files = l[2:].strip().split(' ')
if len(files) == 1:
files = l[2:].strip().split('\t')
src = self._stripbasepath(files[0])
dst = self._stripbasepath(files[1])
if not self._exclude(src) and not self._exclude(dst):
self.changes[rev].ren_files[src] = dst
# Conversion from file to link or from link to file (modified)
elif l.startswith('ch'):
file = self._stripbasepath(l[2:].strip())
if not self._exclude(file):
self.changes[rev].mod_files.append(file)
# Renamed directory
elif l.startswith('/>'):
dirs = l[2:].strip().split(' ')
if len(dirs) == 1:
dirs = l[2:].strip().split('\t')
src = self._stripbasepath(dirs[0])
dst = self._stripbasepath(dirs[1])
if not self._exclude(src) and not self._exclude(dst):
self.changes[rev].ren_dirs[src] = dst
class Pop3Provider(ProviderUtilsBuilder):
__default = None
def __init__(self):
if Pop3Provider.__default:
raise Pop3Provider.__default
ProviderUtilsBuilder.__init__(self, "Pop3")
self.parser = EmailParser()
@staticmethod
def get_instance():
if not Pop3Provider.__default:
Pop3Provider.__default = Pop3Provider()
return Pop3Provider.__default
def load_account(self, props):
acc = AccountCacheMails(props, self)
if not "port" in acc:
acc["port"] = 110
if not "ssl" in acc:
acc["ssl"] = False
return acc
def update_account(self, account):
mbox = self.__connect(account)
messages, new_messages = self.__get_mails(mbox, account)
num_messages = len(new_messages)
max_not = float(
SettingsController.get_instance().get_prefs()["max_notifications"])
account.new_unread = []
for mail_id, mail_num in new_messages:
account.notifications[mail_id] = mail_num
#We only get the e-mail content if all will be shown
if num_messages <= max_not:
msgNum, sub, fr = self.__get_mail_content(mbox, mail_num)
#Store the mail_id, not the msgNum
n = Notification(mail_id, sub, fr)
else:
n = Notification(mail_id, "New mail", "unknow")
account.new_unread.append(n)
#Remove old unread mails not in the current list of unread mails
#TODO Do this better!!!!!
only_current_ids = []
for mail_id, mail_num in messages:
only_current_ids.append(mail_id)
for nid in account.notifications.keys():
if nid not in only_current_ids:
del account.notifications[nid]
mbox.quit()
def get_dialog_def(self):
return [{
"label": "Host",
"type": "str"
}, {
"label": "User",
"type": "str"
}, {
"label": "Password",
"type": "pwd"
}, {
"label": "Port",
"type": "str"
}, {
"label": "Use SSL",
"type": "check"
}]
def populate_dialog(self, widget, acc):
credentials = acc.get_credentials_save()
self._set_text_value("Host", acc["host"])
self._set_text_value("User", credentials.username)
self._set_text_value("Password", credentials.password)
self._set_text_value("Port", str(acc["port"]))
self._set_check_value("Use SSL", utils.get_boolean(acc["ssl"]))
def set_account_data_from_widget(self, account_name, widget, account=None):
host = self._get_text_value("Host")
username = self._get_text_value("User")
password = self._get_text_value("Password")
port = self._get_text_value("Port")
ssl = self._get_check_value("Use SSL")
if host == '' or username == '' or password == '':
raise Exception(
_("The host, user name and the password are mandatory"))
if not account:
props = {
'name': account_name,
'provider_name': self.get_name(),
'host': host,
'port': port,
'ssl': ssl
}
account = self.load_account(props)
else:
account["host"] = host
account["port"] = int(port)
account["ssl"] = ssl
account.set_credentials(Credentials(username, password))
return account
#************** email methods **************
def __connect(self, account):
credentials = account.get_credentials()
port = 110
if "port" in account:
port = int(float(account["port"]))
if not utils.get_boolean(account["ssl"]):
mbox = poplib.POP3(account["host"], port)
else:
mbox = poplib.POP3_SSL(account["host"], port)
mbox.user(credentials.username)
mbox.pass_(credentials.password)
return mbox
def __get_mails(self, mbox, account):
""" Returns:
[list of [msgId, msgNum] all mails, list of [msgId, msgNum] new mails"""
new_messages = []
messages = []
ids = mbox.uidl()
max_not = float(
SettingsController.get_instance().get_prefs()["max_notifications"])
for id_pop in ids[1]:
msgNum = int(id_pop.split(" ")[0])
msgId = id_pop.split(" ")[1]
messages.append([msgId, msgNum])
if msgId not in account.notifications:
new_messages.append([msgId, msgNum])
return [messages, new_messages]
def __get_mail_content(self, mbox, msgNum):
# retrieve only the header
st = "\n".join(mbox.top(msgNum, 0)[1])
#print st
#print "----------------------------------------"
msg = self.parser.parsestr(st, True) # header only
sub = utils.mime_decode(msg.get("Subject"))
fr = utils.mime_decode(msg.get("From"))
return [msgNum, sub, fr]
def parse_messages(self):
parser = Parser()
for message_str in self._messages:
message = parser.parsestr(message_str)
self.parse_message(message)
m.user('*****@*****.**')
m.pass_('password')
# Se obtiene el numero de mensajes pendientes y se hace un
# bucle para cada mensaje
numero = len(m.list()[1])
for i in range(numero):
print "Mensaje numero" + str(i + 1)
print "--------------------"
# Se lee el mensaje
response, headerLines, bytes = m.retr(i + 1)
# Se mete todo el mensaje en un unico string
mensaje = '\n'.join(headerLines)
# Se parsea el mensaje
p = Parser()
email = p.parsestr(mensaje)
# Se sacan por pantalla los campos from, to y subject
print "From: " + email["From"]
print "To: " + email["To"]
print "Subject: " + email["Subject"]
# Si es un mensaje compuesto
if (email.is_multipart()):
# bucle para cada parte del mensaje
for part in email.get_payload():
# Se mira el mime type de la parte
tipo = part.get_content_type()
if ("text/plain" == tipo):
# Si es texto plano, se saca por pantalla
print part.get_payload(decode=True)
if ("image/gif" == tipo):
# Si es imagen, se extrae el nombre del fichero
m.user('*****@*****.**')
m.pass_('PASSWORD')
# Se obtiene el numero de mensajes pendientes y se hace un
# bucle para cada mensaje
numero = len(m.list()[1])
for i in range (numero):
print("Mensaje numero"+str(i+1))
print("--------------------")
# Se lee el mensaje
response, headerLines, bytes = m.retr(i+1)
# Se mete todo el mensaje en un unico string
mensaje='\n'.join(headerLines)
# Se parsea el mensaje
p = Parser()
email = p.parsestr(mensaje)
# Se sacan por pantalla los campos from, to y subject
print("From: "+email["From"])
print("To: "+email["To"])
print("Subject: "+email["Subject"])
# Si es un mensaje compuesto
if (email.is_multipart()):
# bucle para cada parte del mensaje
for part in email.get_payload():
# Se mira el mime type de la parte
tipo = part.get_content_type()
if ("text/plain" == tipo):
# Si es texto plano, se saca por pantalla
print(part.get_payload(decode=True))
if ("image/gif" == tipo):
# Si es imagen, se extrae el nombre del fichero
class Pop3Provider(ProviderUtilsBuilder):
__default = None
def __init__(self):
if Pop3Provider.__default:
raise Pop3Provider.__default
ProviderUtilsBuilder.__init__(self, "Pop3")
self.parser = EmailParser()
@staticmethod
def get_instance():
if not Pop3Provider.__default:
Pop3Provider.__default = Pop3Provider()
return Pop3Provider.__default
def load_account(self, props):
acc = AccountCacheMails(props, self)
if not "port" in acc:
acc["port"] = 110
if not "ssl" in acc:
acc["ssl"] = False
return acc
def update_account (self, account):
mbox = self.__connect(account)
messages, new_messages = self.__get_mails(mbox, account)
num_messages = len(new_messages)
max_not = float(SettingsController.get_instance().get_prefs()["max_notifications"])
account.new_unread = []
for mail_id, mail_num in new_messages:
account.notifications[mail_id] = mail_num
#We only get the e-mail content if all will be shown
if num_messages <= max_not:
msgNum, sub, fr = self.__get_mail_content(mbox, mail_num)
#Store the mail_id, not the msgNum
n = Notification(mail_id, sub, fr)
else:
n = Notification(mail_id, "New mail", "unknow")
account.new_unread.append (n)
#Remove old unread mails not in the current list of unread mails
#TODO Do this better!!!!!
only_current_ids = []
for mail_id, mail_num in messages:
only_current_ids.append(mail_id)
for nid in account.notifications.keys():
if nid not in only_current_ids:
del account.notifications[nid]
mbox.quit()
def get_dialog_def (self):
return [{"label": "Host", "type" : "str"},
{"label": "User", "type" : "str"},
{"label": "Password", "type" : "pwd"},
{"label": "Port", "type" : "str"},
{"label": "Use SSL", "type" : "check"}]
def populate_dialog(self, widget, acc):
credentials = acc.get_credentials_save()
self._set_text_value ("Host",acc["host"])
self._set_text_value ("User", credentials.username)
self._set_text_value ("Password", credentials.password)
self._set_text_value ("Port",str(acc["port"]))
self._set_check_value ("Use SSL",utils.get_boolean(acc["ssl"]))
def set_account_data_from_widget(self, account_name, widget, account=None):
host = self._get_text_value ("Host")
username = self._get_text_value ("User")
password = self._get_text_value ("Password")
port = self._get_text_value ("Port")
ssl = self._get_check_value("Use SSL")
if host=='' or username=='' or password=='':
raise Exception(_("The host, user name and the password are mandatory"))
if not account:
props = {'name' : account_name, 'provider_name' : self.get_name(),
'host' : host, 'port' : port, 'ssl' : ssl}
account = self.load_account(props)
else:
account["host"] = host
account["port"] = int(port)
account["ssl"] = ssl
account.set_credentials(Credentials(username, password))
return account
#************** email methods **************
def __connect(self, account):
credentials = account.get_credentials()
port = 110
if "port" in account:
port = int(float(account["port"]))
if not utils.get_boolean(account["ssl"]):
mbox = poplib.POP3(account["host"], port)
else:
mbox = poplib.POP3_SSL(account["host"], port)
mbox.user(credentials.username)
mbox.pass_(credentials.password)
return mbox
def __get_mails(self, mbox, account):
""" Returns:
[list of [msgId, msgNum] all mails, list of [msgId, msgNum] new mails"""
new_messages = []
messages = []
ids = mbox.uidl()
max_not = float(SettingsController.get_instance().get_prefs()["max_notifications"])
for id_pop in ids[1]:
msgNum = int(id_pop.split(" ")[0])
msgId = id_pop.split(" ")[1]
messages.append( [msgId, msgNum] )
if msgId not in account.notifications:
new_messages.append( [msgId, msgNum] )
return [messages, new_messages]
def __get_mail_content(self, mbox, msgNum):
# retrieve only the header
st = "\n".join(mbox.top(msgNum, 0)[1])
#print st
#print "----------------------------------------"
msg = self.parser.parsestr(st, True) # header only
sub = utils.mime_decode(msg.get("Subject"))
fr = utils.mime_decode(msg.get("From"))
return [msgNum, sub, fr]
def process(mlist, msg, msgdata):
#syslog('gpg','GPG decryption module called')
# Nothing to do when all encryption has been disabled.
if mlist.encrypt_policy == 0:
return
plaintext = None
ciphertext = None
sigid = None
sigmsg = None
is_pgpmime = False
# Check: Is inline pgp?
if msg.get_content_type()=='application/pgp' or msg.get_param('x-action')=='pgp-encrypted':
ciphertext = msg.get_payload()
is_pgpmime = False
# Check: Is pgp/mime?
if msg.get_content_type()=='multipart/encrypted' and msg.get_param('protocol')=='application/pgp-encrypted':
if msg.is_multipart():
for submsg in msg.get_payload():
if submsg.get_content_type()=='application/octet-stream':
is_pgpmime = True
ciphertext=submsg.get_payload()
else:
ciphertext = msg.get_payload()
# Some clients send text/plain messages containing PGP-encrypted data :-(
if not msg.is_multipart() and (ciphertext==None) and \
(len(msg.get_payload())>10):
firstline = msg.get_payload().splitlines()[0]
if firstline=='-----BEGIN PGP MESSAGE-----':
syslog('gpg','Encrypted message detected, although MIME type is %s',msg.get_content_type())
is_pgpmime = False
ciphertext = msg.get_payload()
# This section below is the only custom one
if msg.is_multipart() and (ciphertext==None):
msg_str = str(msg.get_payload()[0])
re_pgp_msg = re.search('-----BEGIN PGP MESSAGE-----[\s\S]*?-----END PGP MESSAGE-----', msg_str)
pgp_msg = re_pgp_msg.group(0)
msg.set_payload(pgp_msg)
ciphertext = str(pgp_msg)
# Ciphertext present? Decode
if ciphertext:
gh = GPGUtils.GPGHelper(mlist)
(plaintext,sigid) = gh.decryptMessage(ciphertext)
if plaintext is None:
syslog('gpg','Unable to decrypt GPG dataGD')
raise Errors.RejectMessage, "Unable to decrypt mail!GD"
# Check decryption result
if plaintext:
# Good signature message
if (not isAdministrativeMail(mlist,msg,msgdata)):
if (not sigid is None):
sigmsg = 'Message had a good signature from sender'
if mlist.anonymous_list==0 or mlist.anonymous_list=='No':
sigmsg += ' (key id %s)'%sigid
else:
sigmsg = 'Posting had no valid signature'
# Check transfer type
parser = Parser()
#syslog('gpg','Test: plaintext=%s',plaintext)
tmpmsg = parser.parsestr(plaintext)
#syslog('gpg','Test: plaintext is\n%s\n',plaintext)
#syslog('gpg','Test: Parsed inner message is\n%s\n',tmpmsg.as_string())
if msg.get_content_type()=='application/pgp':
msg.set_type("text/plain")
msg.del_param("x-action")
for i in ('Content-Type','Content-Disposition','Content-Transfer-Encoding'):
if tmpmsg.has_key(i):
if msg.has_key(i):
msg.replace_header(i,tmpmsg.get(i))
else:
msg.add_header(i,tmpmsg.get(i))
#syslog('gpg','Test: Sigline=%s',sigmsg)
if tmpmsg.is_multipart():
#syslog('gpg','Test: Multipart')
msg.set_payload(None)
for i in tmpmsg.get_payload():
msg.attach(i)
if not sigmsg is None:
sigfooter = MIMEText(sigmsg, 'plain', Utils.GetCharSet(mlist.preferred_language))
sigfooter['Content-Disposition'] = 'inline'
msg.attach(sigfooter)
else:
#syslog('gpg','Test: Not multipart')
tmppayload = tmpmsg.get_payload()
if not sigmsg is None:
if not tmppayload.endswith('\n'):
tmppayload += '\n'
tmppayload += '-- \n%s\n' % sigmsg
msg.set_payload(tmppayload)
if not is_pgpmime:
mailclient = ''
if msg.has_key('User-Agent'):
mailclient = msg.get('User-Agent').lower()
# Content-Transfer-Encoding and charset are not standardized...
if mailclient.startswith('mutt'):
msg.set_param('charset','utf-8')
if msg.has_key('Content-Transfer-Encoding'):
msg.replace_header('Content-Transfer-Encoding','utf-8')
else:
msg.add_header('Content-Transfer-Encoding','utf-8')
else:
# Just a wild guess...
msg.set_param('charset','iso-8859-1')
if msg.has_key('Content-Transfer-Encoding'):
msg.replace_header('Content-Transfer-Encoding','8bit')
else:
msg.add_header('Content-Transfer-Encoding','8bit')
#syslog('gpg','Test: Message is now\n%s\n',msg.as_string())
# --- Old Code ---
#if is_pgpmime:
# if tmpmsg.is_multipart():
# msg.set_payload(None)
# for i in tmpmsg.get_payload():
# msg.attach(i)
# if not sigid is None:
# sigfooter = MIMEText(sigmsg, 'plain', Utils.GetCharSet(mlist.preferred_language))
# sigfooter['Content-Disposition'] = 'inline'
# msg.attach(sigfooter)
# else:
# tmppayload = tmpmsg.get_payload()
# if not sigid is None:
# tmppayload += '\n-- \n%s\n' % sigmsg
# msg.set_payload(tmppayload)
#else:
# # Set content header
# #if msg.get_content_type()=='application/pgp':
# # msg.set_type("text/plain")
# #msg.del_param("x-action")
# # Whole decrypted text is content
# tmppayload = tmpmsg.get_payload()
# if not sigid is None:
# tmppayload += '\n-- \n%s\n' % sigmsg
# msg.set_payload(tmppayload)
elif mlist.encrypt_policy==2:
if enforceEncryptPolicy(mlist,msg,msgdata):
syslog('gpg','Throwing RejectMessage exception: Message has to be encryptedGPGDecrypt')
raise Errors.RejectMessage, "Message has to be encrypted!"
else:
syslog('gpg','Accepting unencrypted message')
syslog.syslog(syslog.LOG_DEBUG, "param: " + param)
os.system(param)
return()
syslog.openlog("csfax_helper")
msg = sys.stdin.readlines()
msg = "".join(msg)
#print msg
faxdest = sys.argv[1]
faxfile = tempfile.NamedTemporaryFile()
errormsgs = ""
hassubject = False
parser = EmailParser()
content = parser.parsestr(msg)
#print content.get_payload()
if content.get("Subject") :
syslog.syslog(syslog.LOG_DEBUG, "has subject header")
hassubject = True
if content.is_multipart() :
syslog.syslog(syslog.LOG_DEBUG, "is multipart")
for mime_part in content.walk() :
syslog.syslog(syslog.LOG_DEBUG, mime_part.get_content_type())
if mime_part.get_content_type() == "application/pdf" :
# print mime_part.get_payload(decode=True)
handle_content_pdf(mime_part.get_payload(decode=True),faxfile)
continue
elif mime_part.get_content_type() == "text/plain" :
class lector_email():
"""Clase utilizada para leer los emails de comandos"""
def __init__(self, usuario, clave):
"""Se establece conexion con el servidor pop de gmail"""
self.usuario = usuario
self.clave = clave
self.conectar()
self.codigos = []
self.codigos.append("ok") #Definimos un codigo inicial y basico
def conectar(self):
"""Se conecta con el servidor pop3"""
self.m = poplib.POP3_SSL('pop.gmail.com', 995)
self.m.user(self.usuario)
self.m.pass_(self.clave)
print "conectado al server pop3"
def definir_mensaje(self, nuevo_mensaje):
"""Agrega nuevo_mensaje a la lista sef.codigo"""
self.codigos.append(nuevo_mensaje.lower())
def cerrar_conexion(self):
"""Encargado de cerrar la coneccion"""
self.m.quit()
def actualizar(self):
"""Ontiene los datos del servidor"""
print "llamada a actualizar"
# Se obtiene el numero de mensajes pendientes y se hace un
# bucle para cada mensaje
self.numero = len(self.m.list()[1])
for i in range(self.numero):
# print "Mensaje numero"+str(i+1)
# print "--------------------"
# Se lee el mensaje
self.response, self.headerLines, self.bytes = self.m.retr(i + 1)
# Se mete todo el mensaje en un unico string
self.mensaje = '\n'.join(self.headerLines)
# Se parsea el mensaje
self.p = Parser()
self.email = self.p.parsestr(self.mensaje)
# Se sacan por pantalla los campos from, to y subject
print "From: " + self.email["From"]
print "To: " + self.email["To"]
print "Subject: " + self.email["Subject"]
print self.email.get_content_type()
print self.email.get_payload(decode=True)
print self.email.is_multipart()
self.tipo = self.email.get_content_type()
for codigo in self.codigos:
if ("text/plain" == self.tipo) and (self.email.get_payload(
decode=True).lower() == codigo):
# Si es texto plano, se escribe en pantalla
print self.email.get_payload(decode=True)
self.m.dele(
i + 1
) #le sumamos 1 ya que python cuenta desde 0 y los emails desde 1
self.cerrar_conexion(
) #nos desconectamos solo si borramos algo, y lo borramos solo si es un mensaje de "ok"
self.conectar(
) #y de esta forma se guardan los archivos en el server
return codigo #retorna el comando que se encontro en el e-mail
return False #retorna falso en caso de no encontrar nada
def process(self):
logging.info("Attempting to process: %s" % datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S"))
imapconn = imaplib.IMAP4_SSL(self.emailserver, self.emailport)
smtpconn = smtplib.SMTP(self.smtpserver, self.smtpport)
smtpconn.ehlo()
smtpconn.starttls()
smtpconn.ehlo()
try:
imapconn.login(self.emaillogin, self.emailpassword)
smtpconn.login(self.emaillogin, self.emailpassword)
except:
imapconn.close()
imapconn.logout()
smtpconn.quit()
logging.info("habwatch Email Login Problem")
return
try:
status, count = imapconn.select(self.emailbox)
except:
imapconn.close()
imapconn.logout()
smtpconn.quit()
logging.info("habwatch Email Select Problem")
return
try:
status, messages = imapconn.search(None, "(UNSEEN)")
except:
imapconn.close()
imapconn.logout()
smtpconn.quit()
logging.info("habwatch Email Search Problem")
return
deleteme = []
if status == "OK":
p = Parser()
for message in messages[0].split():
emailaddy = ""
emaildt = ""
emaildate = ""
emailhtml = ""
emailsubject = ""
emailtext = ""
emaillocation = ""
imgfilename = ""
imgpart = ""
username = ""
logging.info("Processing: %s" % message)
try:
status, data = imapconn.fetch(message, "(BODY[HEADER.FIELDS (DATE SUBJECT FROM)])")
except:
imapconn.close()
imapconn.logout()
smtpconn.quit()
logging.info("habwatch Email Fetch Problem")
return
msg = p.parsestr(data[0][1])
when = msg.__getitem__("Date")
who = msg.__getitem__("From")
subject = msg.__getitem__("Subject")
matchemail = re.compile(r"[\w\-][\w\-\.]+@[\w\-][\w\-\.]+[a-zA-Z]{1,4}")
emailaddy = matchemail.findall(who)[0]
username = self.getusername(emailaddy)
# take care of stupid no leading zero problem
if when[6] == " ":
when = when[0:5] + "0" + when[5:]
emaildate = datetime.datetime(*time.strptime(when[5:16], "%d %b %Y")[0:6]).strftime("%Y-%m-%d")
logging.info("when: %s" % when)
emaildt = datetime.datetime(*time.strptime(when[5:25], "%d %b %Y %H:%M:%S")[0:6]).strftime(
"%Y-%m-%d %H:%M:%S"
)
emailsubject = subject
try:
status, data = imapconn.fetch(message, "(RFC822)")
except:
imapconn.close()
imapconn.logout()
smtpconn.quit()
logging.info("habwatch Email Fetch 2 Problem")
return
emailbody = data[0][1]
validtext = False
validhtml = False
validimg = False
emailmsg = email.message_from_string(emailbody)
for part in emailmsg.walk():
type = part.get_content_type()
payload = part.get_payload()
payload = part.get_payload(decode=1)
filename = part.get_filename()
multipart = part.is_multipart()
# TYPES INCLUDE:
# type multipart/mixed
# type multipart/alternative
# type text/plain
# type text/html
# type image/jpeg
if multipart:
continue
if type == "text/plain":
emailtext = payload
validtext = True
if type == "text/html":
emailhtml = payload
validhtml = True
if (type == "image/gif") or (type == "image/jpeg") or (type == "image/png"):
if validimg == True:
continue
else:
imgpart = part
imgfilename = filename
logging.info("imgfilename: %s" % imgfilename)
validimg = True
# Just one image per user son!
if validimg:
break
logging.info("Username: %s" % username)
logging.info("Email Address: %s" % emailaddy)
logging.info("Email DT: %s" % emaildt)
logging.info("Email Date: %s" % emaildate)
logging.info("Email Subject: %s" % emailsubject)
logging.info("Email Text: %s" % emailtext)
logging.info("Email HTML: %s" % emailhtml)
logging.info("IMG FileName: %s" % imgfilename)
# Send it out
if validimg and (not validate.spam(emailaddy)):
try:
msg = MIMEMultipart()
msg["From"] = self.emaillogin
msg["To"] = self.smtpaddress
msg["Date"] = formatdate(localtime=True)
# replace this
msg["Subject"] = "photo tags: %s %s" % (username, subject or "")
msg.attach(MIMEText("%s\n\n- Camera phone upload powered by HAB Watch" % emailtext))
msg.attach(imgpart)
smtpconn.sendmail(self.emaillogin, self.smtpaddress, msg.as_string())
logging.info("Sent to Flickr")
# Change this so that things get deleted or not
deleteme.append(message)
logging.info("%s %s %s sent to Flickr" % (username, emailaddy, emaildt))
except:
logging.info("Exception to Sending Email")
pass
else:
logging.info("%s %s %s deleted" % (username, emailaddy, emaildt))
deleteme.append(message)
logging.info("Invalid Img or User")
if deleteme:
deleteme.sort()
deleteme.reverse()
for message in deleteme:
try:
imapconn.store(message, "+FLAGS", "(\\Deleted)")
imapconn.expunge()
except:
imapconn.close()
imapconn.logout()
smtpconn.quit()
logging.info("habwatch Email Delete Problem")
return
imapconn.close()
imapconn.logout()
smtpconn.quit()
class EmailParser(object):
"""
Parse email to extract usefull informations
"""
def __init__(self):
self._parser = Parser()
self._templates = self.__load_templates()
def parse(self, content):
"""
Parse a raw email
:param str content: The raw email
:rtype: `worker.parsing.parser.ParsedEmail`
:returns: The parsed email
"""
parsed_email = ParsedEmail()
for att in ['category', 'ips', 'urls', 'fqdn', 'logs']:
setattr(parsed_email, att, None)
headers = self.get_headers(content)
body, attachments = self.get_body_and_attachments(content)
setattr(parsed_email, 'body', body)
setattr(parsed_email, 'attachments', attachments)
setattr(parsed_email, 'date', get_date_from_headers(headers))
setattr(parsed_email, 'received', get_received_from_headers(headers))
setattr(parsed_email, 'provider', get_sender_from_headers(headers))
setattr(parsed_email, 'recipients', get_recipients_from_headers(headers))
setattr(parsed_email, 'subject', get_subject_from_headers(headers))
setattr(parsed_email, 'trusted', is_provider_trusted(headers))
setattr(parsed_email, 'ack', is_email_ack(parsed_email.provider, parsed_email.subject, body))
content_to_parse = parsed_email.subject + '\n' + parsed_email.body
templates = get_ordered_template_names_list(parsed_email, content_to_parse)
# Finally order is [recipients, provider, keywords, default]
for name in templates:
template = self.get_template(name)
if not template:
continue
self.update_parsed_email(parsed_email, content_to_parse, template)
if any((parsed_email.urls, parsed_email.ips, parsed_email.fqdn)) or template.get('fallback') is False:
break
# Checking if a default category is set for this provider
try:
prov = Provider.objects.get(email=parsed_email.provider)
parsed_email.category = prov.defaultCategory_id if prov.defaultCategory_id else parsed_email.category
except (KeyError, ObjectDoesNotExist):
pass
# If no category, checking if provider email match a generic provider
if parsed_email.category not in CATEGORIES:
for reg, val in regexp.PROVIDERS_GENERIC.iteritems(): # For 123854689@spamcop for example
if reg.match(parsed_email.provider):
try:
prov = Provider.objects.get(email=val)
parsed_email.category = prov.defaultCategory_id if prov.defaultCategory_id else DEFAULT_CATEGORY
break
except (KeyError, ObjectDoesNotExist):
parsed_email.category = DEFAULT_CATEGORY
break
# Still not ?
if parsed_email.category not in CATEGORIES:
parsed_email.category = DEFAULT_CATEGORY
# if Provider provide link to online logs, get it
if parsed_email.logs:
logs = get_online_logs(parsed_email.logs)
if logs:
parsed_email.attachments.append(logs)
clean_parsed_email_items(parsed_email)
return parsed_email
def get_headers(self, raw):
"""
Returns email SMTP headers
:param str raw: The raw email
"""
return self._parser.parsestr(raw, headersonly=True)
def get_body_and_attachments(self, raw):
"""
Get the body of the mail and retreive attachments
:param str raw: The raw email
:rtype: tuple
:returns: The decoded body and a list of attachments
"""
messages = self._parser.parsestr(raw)
attachments = []
body = ''
for message in messages.walk():
content_type = message.get_content_type().lower()
if message.is_multipart() and content_type != 'message/rfc822':
continue
content_disposition = message.get('Content-Disposition')
if content_disposition:
content_disposition = content_disposition.decode('utf-8').encode('utf-8')
if content_disposition:
disposition = content_disposition.decode('utf-8').strip().split(';')[0].lower()
if disposition == 'attachment':
attachments.append(parse_attachment(message))
continue
elif disposition == 'inline' and content_type not in ['text/html', 'text/plain']:
attachments.append(parse_attachment(message))
continue
content = message.get_payload(decode=True)
if not content:
content = message.as_string()
if content:
body += utils.decode_every_charset_in_the_world(content, message.get_content_charset())
return body, attachments
def get_template(self, provider):
"""
Mail from trusted provider (spamhaus etc ...) are formatted
For each provider, regex pattern extract useful informations
The set of patterns are named template
URL, IPS ...
:param str provider: The email of the provider
:rtype: dict:
:returns: The corresponding parsing template if exists
"""
template = None
try:
template = self._templates[provider]
except KeyError:
for reg, val in regexp.PROVIDERS_GENERIC.iteritems():
if reg.match(provider):
try:
template = self._templates[val]
return template
except KeyError:
pass
return template
def update_parsed_email(self, parsed_email, content, template):
"""
Get all items (IP, URL) of a parsed_email
"""
try:
for key, val in template['regexp'].iteritems():
if 'pattern' in val:
if 'pretransform' in val:
res = re.findall(val['pattern'], val['pretransform'](content), re.IGNORECASE)
else:
res = re.findall(val['pattern'], content, re.IGNORECASE)
if res:
if 'transform' in val:
res = self.__transform(res[0])
setattr(parsed_email, key, res)
elif 'value' in val:
setattr(parsed_email, key, val['value'])
except AttributeError:
pass
def __load_templates(self):
"""
Loads provider template from TEMPLATE_DIR
:rtype: dict:
:returns: All templates available in TEMPLATE_DIR
"""
template_base = os.path.dirname(os.path.realpath(__file__)) + TEMPLATE_DIR
modules = glob.glob(os.path.join(template_base, '*.py'))
template_files = [os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')]
templates = {}
for template in template_files:
infos = imp.load_source(template, os.path.join(template_base, template + '.py'))
templates[infos.TEMPLATE.pop('email')] = infos.TEMPLATE
return templates
def __transform(self, content):
"""
Try to get parsed_email category with defined keywords
:param str content: The content to parse
:rtype: str
:returns: The category (or None if not identified)
"""
if isinstance(content, tuple):
text = content[0]
else:
text = content
category = None
last_count = 0
for cat, pattern in regexp.CATEGORY.iteritems():
count = len(re.findall(pattern, text, re.IGNORECASE))
if count > last_count:
last_count = count
category = cat
return category
class gnuarch_source(converter_source, commandline):
class gnuarch_rev(object):
def __init__(self, rev):
self.rev = rev
self.summary = ''
self.date = None
self.author = ''
self.continuationof = None
self.add_files = []
self.mod_files = []
self.del_files = []
self.ren_files = {}
self.ren_dirs = {}
def __init__(self, ui, path, revs=None):
super(gnuarch_source, self).__init__(ui, path, revs=revs)
if not os.path.exists(os.path.join(path, '{arch}')):
raise NoRepo(_("%s does not look like a GNU Arch repository")
% path)
# Could use checktool, but we want to check for baz or tla.
self.execmd = None
if util.findexe('baz'):
self.execmd = 'baz'
else:
if util.findexe('tla'):
self.execmd = 'tla'
else:
raise util.Abort(_('cannot find a GNU Arch tool'))
commandline.__init__(self, ui, self.execmd)
self.path = os.path.realpath(path)
self.tmppath = None
self.treeversion = None
self.lastrev = None
self.changes = {}
self.parents = {}
self.tags = {}
self.catlogparser = Parser()
self.encoding = encoding.encoding
self.archives = []
def before(self):
# Get registered archives
self.archives = [i.rstrip('\n')
for i in self.runlines0('archives', '-n')]
if self.execmd == 'tla':
output = self.run0('tree-version', self.path)
else:
output = self.run0('tree-version', '-d', self.path)
self.treeversion = output.strip()
# Get name of temporary directory
version = self.treeversion.split('/')
self.tmppath = os.path.join(tempfile.gettempdir(),
'hg-%s' % version[1])
# Generate parents dictionary
self.parents[None] = []
treeversion = self.treeversion
child = None
while treeversion:
self.ui.status(_('analyzing tree version %s...\n') % treeversion)
archive = treeversion.split('/')[0]
if archive not in self.archives:
self.ui.status(_('tree analysis stopped because it points to '
'an unregistered archive %s...\n') % archive)
break
# Get the complete list of revisions for that tree version
output, status = self.runlines('revisions', '-r', '-f', treeversion)
self.checkexit(status, 'failed retrieving revisions for %s'
% treeversion)
# No new iteration unless a revision has a continuation-of header
treeversion = None
for l in output:
rev = l.strip()
self.changes[rev] = self.gnuarch_rev(rev)
self.parents[rev] = []
# Read author, date and summary
catlog, status = self.run('cat-log', '-d', self.path, rev)
if status:
catlog = self.run0('cat-archive-log', rev)
self._parsecatlog(catlog, rev)
# Populate the parents map
self.parents[child].append(rev)
# Keep track of the current revision as the child of the next
# revision scanned
child = rev
# Check if we have to follow the usual incremental history
# or if we have to 'jump' to a different treeversion given
# by the continuation-of header.
if self.changes[rev].continuationof:
treeversion = '--'.join(
self.changes[rev].continuationof.split('--')[:-1])
break
# If we reached a base-0 revision w/o any continuation-of
# header, it means the tree history ends here.
if rev[-6:] == 'base-0':
break
def after(self):
self.ui.debug('cleaning up %s\n' % self.tmppath)
shutil.rmtree(self.tmppath, ignore_errors=True)
def getheads(self):
return self.parents[None]
def getfile(self, name, rev):
if rev != self.lastrev:
raise util.Abort(_('internal calling inconsistency'))
if not os.path.lexists(os.path.join(self.tmppath, name)):
return None, None
return self._getfile(name, rev)
def getchanges(self, rev, full):
if full:
raise util.Abort(_("convert from arch do not support --full"))
self._update(rev)
changes = []
copies = {}
for f in self.changes[rev].add_files:
changes.append((f, rev))
for f in self.changes[rev].mod_files:
changes.append((f, rev))
for f in self.changes[rev].del_files:
changes.append((f, rev))
for src in self.changes[rev].ren_files:
to = self.changes[rev].ren_files[src]
changes.append((src, rev))
changes.append((to, rev))
copies[to] = src
for src in self.changes[rev].ren_dirs:
to = self.changes[rev].ren_dirs[src]
chgs, cps = self._rendirchanges(src, to)
changes += [(f, rev) for f in chgs]
copies.update(cps)
self.lastrev = rev
return sorted(set(changes)), copies, set()
def getcommit(self, rev):
changes = self.changes[rev]
return commit(author=changes.author, date=changes.date,
desc=changes.summary, parents=self.parents[rev], rev=rev)
def gettags(self):
return self.tags
def _execute(self, cmd, *args, **kwargs):
cmdline = [self.execmd, cmd]
cmdline += args
cmdline = [util.shellquote(arg) for arg in cmdline]
cmdline += ['>', os.devnull, '2>', os.devnull]
cmdline = util.quotecommand(' '.join(cmdline))
self.ui.debug(cmdline, '\n')
return os.system(cmdline)
def _update(self, rev):
self.ui.debug('applying revision %s...\n' % rev)
changeset, status = self.runlines('replay', '-d', self.tmppath,
rev)
if status:
# Something went wrong while merging (baz or tla
# issue?), get latest revision and try from there
shutil.rmtree(self.tmppath, ignore_errors=True)
self._obtainrevision(rev)
else:
old_rev = self.parents[rev][0]
self.ui.debug('computing changeset between %s and %s...\n'
% (old_rev, rev))
self._parsechangeset(changeset, rev)
def _getfile(self, name, rev):
mode = os.lstat(os.path.join(self.tmppath, name)).st_mode
if stat.S_ISLNK(mode):
data = os.readlink(os.path.join(self.tmppath, name))
if mode:
mode = 'l'
else:
mode = ''
else:
data = open(os.path.join(self.tmppath, name), 'rb').read()
mode = (mode & 0o111) and 'x' or ''
return data, mode
def _exclude(self, name):
exclude = ['{arch}', '.arch-ids', '.arch-inventory']
for exc in exclude:
if name.find(exc) != -1:
return True
return False
def _readcontents(self, path):
files = []
contents = os.listdir(path)
while len(contents) > 0:
c = contents.pop()
p = os.path.join(path, c)
# os.walk could be used, but here we avoid internal GNU
# Arch files and directories, thus saving a lot time.
if not self._exclude(p):
if os.path.isdir(p):
contents += [os.path.join(c, f) for f in os.listdir(p)]
else:
files.append(c)
return files
def _rendirchanges(self, src, dest):
changes = []
copies = {}
files = self._readcontents(os.path.join(self.tmppath, dest))
for f in files:
s = os.path.join(src, f)
d = os.path.join(dest, f)
changes.append(s)
changes.append(d)
copies[d] = s
return changes, copies
def _obtainrevision(self, rev):
self.ui.debug('obtaining revision %s...\n' % rev)
output = self._execute('get', rev, self.tmppath)
self.checkexit(output)
self.ui.debug('analyzing revision %s...\n' % rev)
files = self._readcontents(self.tmppath)
self.changes[rev].add_files += files
def _stripbasepath(self, path):
if path.startswith('./'):
return path[2:]
return path
def _parsecatlog(self, data, rev):
try:
catlog = self.catlogparser.parsestr(data)
# Commit date
self.changes[rev].date = util.datestr(
util.strdate(catlog['Standard-date'],
'%Y-%m-%d %H:%M:%S'))
# Commit author
self.changes[rev].author = self.recode(catlog['Creator'])
# Commit description
self.changes[rev].summary = '\n\n'.join((catlog['Summary'],
catlog.get_payload()))
self.changes[rev].summary = self.recode(self.changes[rev].summary)
# Commit revision origin when dealing with a branch or tag
if 'Continuation-of' in catlog:
self.changes[rev].continuationof = self.recode(
catlog['Continuation-of'])
except Exception:
raise util.Abort(_('could not parse cat-log of %s') % rev)
def _parsechangeset(self, data, rev):
for l in data:
l = l.strip()
# Added file (ignore added directory)
if l.startswith('A') and not l.startswith('A/'):
file = self._stripbasepath(l[1:].strip())
if not self._exclude(file):
self.changes[rev].add_files.append(file)
# Deleted file (ignore deleted directory)
elif l.startswith('D') and not l.startswith('D/'):
file = self._stripbasepath(l[1:].strip())
if not self._exclude(file):
self.changes[rev].del_files.append(file)
# Modified binary file
elif l.startswith('Mb'):
file = self._stripbasepath(l[2:].strip())
if not self._exclude(file):
self.changes[rev].mod_files.append(file)
# Modified link
elif l.startswith('M->'):
file = self._stripbasepath(l[3:].strip())
if not self._exclude(file):
self.changes[rev].mod_files.append(file)
# Modified file
elif l.startswith('M'):
file = self._stripbasepath(l[1:].strip())
if not self._exclude(file):
self.changes[rev].mod_files.append(file)
# Renamed file (or link)
elif l.startswith('=>'):
files = l[2:].strip().split(' ')
if len(files) == 1:
files = l[2:].strip().split('\t')
src = self._stripbasepath(files[0])
dst = self._stripbasepath(files[1])
if not self._exclude(src) and not self._exclude(dst):
self.changes[rev].ren_files[src] = dst
# Conversion from file to link or from link to file (modified)
elif l.startswith('ch'):
file = self._stripbasepath(l[2:].strip())
if not self._exclude(file):
self.changes[rev].mod_files.append(file)
# Renamed directory
elif l.startswith('/>'):
dirs = l[2:].strip().split(' ')
if len(dirs) == 1:
dirs = l[2:].strip().split('\t')
src = self._stripbasepath(dirs[0])
dst = self._stripbasepath(dirs[1])
if not self._exclude(src) and not self._exclude(dst):
self.changes[rev].ren_dirs[src] = dst
def decryptGpg(mlist, msg, msgdata):
"""Returns (encrypted (bool), signed (bool), key_ids), msg is replaced with
decrypted msg"""
encrypted = False
signed = False
key_ids = []
plaintext = None
ciphertext = None
is_pgpmime = False
# Check: Is inline pgp?
if msg.get_content_type()=='application/pgp' or msg.get_param('x-action')=='pgp-encrypted':
ciphertext = msg.get_payload()
is_pgpmime = False
# Check: Is pgp/mime?
if msg.get_content_type()=='multipart/encrypted' and msg.get_param('protocol')=='application/pgp-encrypted':
if msg.is_multipart():
for submsg in msg.get_payload():
if submsg.get_content_type()=='application/octet-stream':
is_pgpmime = True
ciphertext = submsg.get_payload()
else:
ciphertext = msg.get_payload()
# Some clients send text/plain messages containing PGP-encrypted data :-(
if not msg.is_multipart() and (ciphertext==None) and \
(len(msg.get_payload())>10):
firstline = msg.get_payload().splitlines()[0]
if firstline=='-----BEGIN PGP MESSAGE-----':
syslog('gpg','Encrypted message detected, although MIME type is %s',msg.get_content_type())
is_pgpmime = False
ciphertext = msg.get_payload()
# Ciphertext present? Decode
if ciphertext:
gh = GPGUtils.GPGHelper(mlist)
(plaintext,key_ids) = gh.decryptMessage(ciphertext)
if plaintext is None:
syslog('gpg','Unable to decrypt GPG data')
raise Errors.RejectMessage, "Unable to decrypt mail!"
else:
encrypted = True
if key_ids:
signed = True
if not encrypted:
return (encrypted, signed, key_ids)
# Check decryption result
# Check transfer type
parser = Parser()
tmpmsg = parser.parsestr(plaintext)
if msg.get_content_type()=='application/pgp':
msg.set_type("text/plain")
msg.del_param("x-action")
for i in ('Content-Type','Content-Disposition','Content-Transfer-Encoding'):
if tmpmsg.has_key(i):
if msg.has_key(i):
msg.replace_header(i,tmpmsg.get(i))
else:
msg.add_header(i,tmpmsg.get(i))
if tmpmsg.is_multipart():
msg.set_payload(None)
for i in tmpmsg.get_payload():
msg.attach(i)
else:
tmppayload = tmpmsg.get_payload()
msg.set_payload(tmppayload)
if not is_pgpmime:
mailclient = ''
if msg.has_key('User-Agent'):
mailclient = msg.get('User-Agent').lower()
# Content-Transfer-Encoding and charset are not standardized...
if mailclient.startswith('mutt'):
msg.set_param('charset','utf-8')
if msg.has_key('Content-Transfer-Encoding'):
msg.replace_header('Content-Transfer-Encoding','utf-8')
else:
msg.add_header('Content-Transfer-Encoding','utf-8')
else:
# Just a wild guess...
msg.set_param('charset','iso-8859-1')
if msg.has_key('Content-Transfer-Encoding'):
msg.replace_header('Content-Transfer-Encoding','8bit')
else:
msg.add_header('Content-Transfer-Encoding','8bit')
if encrypted:
msg.add_header('X-Mailman-SLS-decrypted', 'Yes')
return (encrypted, signed, key_ids)
def processMails(self, text, att_file):
"""
Parse mail for display in XBMC
"""
myemail = email.message_from_string(text)
p = EmailParser()
msgobj = p.parsestr(text)
if msgobj['Subject'] is not None:
decodefrag = decode_header(msgobj['Subject'])
subj_fragments = []
for s , enc in decodefrag:
if enc:
s = unicode(s , enc).encode('utf8','replace')
subj_fragments.append(s)
subject = ''.join(subj_fragments)
else:
subject = None
if msgobj['Date'] is not None:
date = msgobj['Date']
else:
date = '--'
Sujet = subject
realname = parseaddr(msgobj.get('From'))[1]
body = None
html = None
for part in msgobj.walk():
content_disposition = part.get("Content-Disposition", None)
prog = re.compile('attachment')
#Retrouve le nom des fichiers attaches
if prog.search(str(content_disposition)):
file_att = str(content_disposition)
pattern = Pattern(r"\"(.+)\"")
att_file += str(pattern.findall(file_att))
if part.get_content_type() == "text/plain":
if body is None:
body = ""
try :
#Si pas de charset défini
if (part.get_content_charset() is None):
body += part.get_payload(decode=True)
else:
body += unicode(
part.get_payload(decode=True),
part.get_content_charset(),
'replace'
).encode('utf8','replace')
except Exception, e:
body += "Erreur unicode"
print "BODY = %s " % body
elif part.get_content_type() == "text/html":
if html is None:
html = ""
try :
unicode_coded_entities_html = unicode(BeautifulStoneSoup(html,
convertEntities=BeautifulStoneSoup.HTML_ENTITIES))
html += unicode_coded_entities_html
html = html2text(html)
except Exception, e:
html += "Erreur unicode html"
def recibir_facturas(self, cr, uid, ids, context=None):
revisados = self.pool.get("messages_server")
facturas = self.pool.get("account.invoice")
lineas = self.pool.get("account.invoice.line")
partners = self.pool.get("res.partner")
rechazo_obj = self.pool.get("ae.rechazadas")
for ir in self.browse(cr, uid, ids, context):
#conectando
if ir.i_layer is True:
m = poplib.POP3_SSL(ir.i_server, ir.i_port)
else:
m = poplib.POP3(ir.i_server, ir.i_port)
m.user(ir.i_user)
m.pass_(ir.i_pass)
#leyendo los mails
mensajes = len(m.list()[1])
for i in range(mensajes):
response, headerLines, bytes = m.retr(i + 1)
mensaje = '\n'.join(headerLines)
p = Parser()
email = p.parsestr(mensaje)
if email.is_multipart():
part_count = 0
for part in email.get_payload():
part_count = part_count + 1
tipo = part.get_content_type()
if tipo == "application/xml" or tipo == "text/xml":
try:
fp = open('fact_process.xml', 'wb')
fp.write(part.get_payload(decode=True))
fp.close()
doc = minidom.parse('fact_process.xml')
except:
rechazo_obj.create(cr, uid, {
'name': "XML No Válido",
'descripcion': email
})
continue
tagFolio = "tfd:TimbreFiscalDigital"
foli = ""
try:
folio = 0
folio = doc.getElementsByTagName(tagFolio)[0]
foli = folio.getAttribute('UUID')
print(("Folio de la factura" + str(foli)))
except:
rechazo_obj.create(cr, uid, {
'name': "UUID No Encontado",
'descripcion': doc
})
print("Rompiendo el cicle X")
continue
#print(lFolio)
try:
tagEmpresa = 'cfdi:Receptor'
empresa = doc.getElementsByTagName(tagEmpresa)[0]
nempresa = empresa.getAttribute('rfc')
print(("Nombre empresa " + nempresa))
u_obj = self.pool.get("res.users")
print("Aqui 1")
con = False
print("Aqui 2")
print(("User id " + str(uid)))
print("Aqui 3")
for cp in u_obj.browse(cr, uid, [uid]):
print("Aqui 4")
print(("Probando " + nempresa + "VS" +
cp.company_id.rfc))
print((str(nempresa != cp.company_id.rfc)))
print("Aqui 4")
if nempresa != cp.company_id.rfc:
print("Broken")
rechazo_obj.create(cr, uid, {
'name': "Empresas no coinciden",
'descripcion': nempresa + " VS " +
cp.company_id.name
})
con = True
if con:
print("Rompido")
continue
except:
print("Aqui se rompio")
rechazo_obj.create(cr, uid, {
'name': "Empresa no coincide",
'descripcion': email
})
continue
#revisar que no existe ya el registro
print("Aqui 5")
idm = revisados.search(cr, uid,
[('name', '=', foli),
('cancelada', '=', False)])
print("Aqui 6")
ids_e = len(revisados.browse(cr, uid, idm, context))
print(ids_e)
print("Aqui 7")
if ids_e == 0:
print("Creando Factura")
#leyendo la fecha
tagComprobante = 'cfdi:Comprobante'
fe = doc.getElementsByTagName(tagComprobante)[0]
fecha = fe.getAttribute('fecha')
print(('Fecha: ' + fecha[0:10]))
iva = 0.0
riva = 0.0
risr = 0.0
try:
print("Buscando Impuestos")
ti = 'cfdi:Traslado'
liva = doc.getElementsByTagName(ti)[0]
print("Existe el IVA 0")
siva = str(liva.getAttribute('importe'))
print(("Valor del IVA" + siva))
print((type(siva)))
print((float(siva)))
print("X")
iva = float(siva)
print(("EL IVA ES POR " + str(iva)))
print("Mensaje x")
ti2 = 'cfdi:Retencion'
elemento = doc.getElementsByTagName(ti2)[0]
print(("Que raro" + str(elemento)))
for lret in doc.getElementsByTagName(ti2):
print("Retención encontrada")
impuesto = lret.getAttribute('impuesto')
print(("Es XX " + impuesto))
if impuesto == "IVA":
print("Es IVA")
riva = float(lret.getAttribute(
'importe'))
print(("Retencion " + str(riva)))
print("No Es IVA")
if impuesto == "ISR":
print("Es ISR")
risr = float(lret.getAttribute(
'importe'))
print(("Retencion " + str(risr)))
print("Sali del ciclo")
except:
print("Impuesto no encontrado")
#leyendo el RFC de quien emite la factura
print("Continuando")
tagEmisor = 'cfdi:Emisor'
emisor = doc.getElementsByTagName(tagEmisor)[0]
rfc = emisor.getAttribute('rfc')
print(("Proveedr: " + rfc))
#buscando al cliente por rfc
id_proveedor = 0
print("Aqui 8")
par = partners.search(cr, uid,
[('vat', '=', rfc)])
print("Aqui 9")
iP = len(partners.browse(cr, uid, par, context))
print("Aqui 10")
if iP == 0:
print("Aqui 11")
rfc = "MX" + rfc
print("Aqui 12")
par = partners.search(cr, uid,
[('vat', '=', rfc)])
print("Aqui 13")
iP = len(partners.browse(cr, uid, par,
context))
if iP == 0:
mensaje = 'No existe el proveedor con RFC'
print(('Error de Proveedor' + mensaje
+ ': ' + rfc))
rechazo_obj.create(cr, uid, {
'name': emisor,
'rfc': rfc,
'descripcion': "RFC No Encontrado"
})
continue
for i in partners.browse(cr, uid, par, context):
id_proveedor = i.id
#creando la factura
id_fact = facturas.create(cr, uid,
{
'partner_id': id_proveedor,
'date_invoice': fecha[0:10],
'account_id': 209,
'journal_id': 2,
'type': 'in_invoice',
'reference_type': 'none',
'currency_id': 34,
'iva': iva,
'retiva': riva,
'retisr': risr,
},
context)
#leyendo los conceptos de compra
#print("Productos")
tagConcepto = 'cfdi:Concepto'
prods = doc.getElementsByTagName(tagConcepto)
for prod in prods:
cantidad = prod.getAttribute('cantidad')
desc = prod.getAttribute('descripcion')
precio = prod.getAttribute('valorUnitario')
#print("")
#print(" Cant.: " + cantidad)
#print(" Desc.: " + desc.encode("latin1"))
#print(" Precio: " + precio)
#creando las lineas de la factura
lineas.create(cr, uid,
{
'invoice_id': id_fact,
'name': desc,
'quantity': cantidad,
'price_unit': precio,
'account_id': 28131,
},
context)
#Colocando los impuestos
for fa in facturas.browse(cr, uid, [id_fact]):
total = fa.amount_total + iva - riva - risr
facturas.write(cr, uid, [fa.id],
{'total': total})
#creando el registro de guardado
revisados.create(cr, uid,
{
'name': foli,
'from': email["From"],
'to': email["To"],
'fact': id_fact,
'fecha': fecha[0:10],
'proveedor': id_proveedor,
},
context)
return True
# ---------------------------------------
def head(self, id=""):
"""Head command. Return value is a Message from email package."""
try:
if isinstance(id, ArticleIdentification):
lst = self.__nntp.head(id.getNumber())
else:
lst = self.__nntp.head(str(id))
except NNTPError, e:
raise NNTPGeneralError, e.response
self.setLastResponse(lst[0])
txt = self.as_text(lst[3])
from email.Parser import Parser
p = Parser()
return p.parsestr(txt, True)
# ---------------------------------------
def body(self, id=""):
"""Body command. Return value is a Message from email package."""
try:
if isinstance(id, ArticleIdentification):
lst = self.__nntp.body(id.getNumber())
else:
lst = self.__nntp.body(str(id))
except NNTPError, e:
raise NNTPGeneralError, e.response
self.setLastResponse(lst[0])
txt = self.as_text(lst[3])
from email.Parser import Parser
def eliminar_movimientos(self, cr, uid, ids, context=None):
revisados = self.pool.get("messages_server")
facturas = self.pool.get("account.invoice")
rechazo_obj = self.pool.get("ae.rechazadas")
move_obj = self.pool.get("account.move")
for ir in self.browse(cr, uid, ids, context):
#conectando
if ir.i_layer is True:
m = poplib.POP3_SSL(ir.i_server, ir.i_port)
else:
m = poplib.POP3(ir.i_server, ir.i_port)
m.user(ir.i_user)
m.pass_(ir.i_pass)
#leyendo los mails
mensajes = len(m.list()[1])
for i in range(mensajes):
response, headerLines, bytes = m.retr(i + 1)
mensaje = '\n'.join(headerLines)
p = Parser()
email = p.parsestr(mensaje)
if email.is_multipart():
part_count = 0
for part in email.get_payload():
part_count = part_count + 1
tipo = part.get_content_type()
if tipo == "application/xml" or tipo == "text/xml":
try:
fp = open('fact_process.xml', 'wb')
fp.write(part.get_payload(decode=True))
fp.close()
doc = minidom.parse('fact_process.xml')
except:
rechazo_obj.create(cr, uid, {
'name': "XML No Válido",
'descripcion': email
})
continue
tagFolio = "tfd:TimbreFiscalDigital"
foli = ""
try:
folio = 0
folio = doc.getElementsByTagName(tagFolio)[0]
foli = folio.getAttribute('UUID')
print(("Folio de la factura" + str(foli)))
except:
rechazo_obj.create(cr, uid, {
'name': "UUID No Encontado",
'descripcion': doc
})
print("Rompiendo el cicle X")
continue
#print(lFolio)
#print("Eiminando" + str(foli))
for ms in revisados.browse(cr, uid,
revisados.search(cr, uid, [('name', '=',
foli)])):
if(ms.fact and ms.fact.move_id):
try:
move_obj.write(cr, uid,
[ms.fact.move_id.id],
{'state': 'draft'})
move_obj.unlink(cr, uid,
[ms.fact.move_id.id])
except:
print("Movimiento no eliminado")
try:
facturas.unlink(cr, uid, [ms.fact.id])
except:
print("Factura no eliminada")
try:
revisados.unlink(cr, uid, [ms.id])
except:
print(("No se elimino " + str(foli)))
return True
