def _meta_from_zipfile(path):
"""Retrieve metadata from a zip file of a distutils distribution.
Returns a dict.
"""
metadata = {}
oldpkginfo = None
zf = zipfile.ZipFile(path, 'r')
for name in zf.namelist():
if '.egg-info/' in name and not name.endswith('.egg-info/'):
metaname = os.path.splitext(os.path.basename(name))[0]
meta = zf.read(name).strip()
if name.endswith('/PKG-INFO'):
instr = StringIO.StringIO(meta)
message = rfc822.Message(instr)
for k, v in message.items():
metadata[k] = v
elif name.endswith('.txt'):
_parse_txt(name, meta, metadata)
elif name.endswith('/not-zip-safe'):
metadata['zip-safe'] = False
elif name.endswith('/zip-safe'):
metadata['zip-safe'] = True
else:
continue
elif name.endswith('/PKG-INFO'):
oldpkginfo = name
# if there's no egg-info, use old PKG-INFO if found
if len(metadata) == 0 and oldpkginfo:
instr = StringIO.StringIO(zf.read(oldpkginfo).strip())
message = rfc822.Message(instr)
for k, v in message.items():
metadata[k] = v
return metadata
class Mailwatcher(Frame):
def __init__(self, master=None):
Frame.__init__(self, master)
self.pack(side=TOP, expand=YES, fill=BOTH)
self.scroll = Scrollbar(self)
self.list = Listbox(self,
font='7x13',
yscrollcommand=self.scroll.set,
setgrid=1,
height=6,
width=80)
self.scroll.configure(command=self.list.yview)
self.scroll.pack(side=LEFT, fill=BOTH)
self.list.pack(side=LEFT, expand=YES, fill=BOTH)
def getmail(self):
self.after(1000 * PollInterval, self.getmail)
self.list.delete(0, END)
try:
M = imaplib.IMAP4(imap_server)
M.login(imap_user, imap_password)
except Exception, e:
self.list.insert(END, 'IMAP login error: ', e)
return
try:
result, message = M.select(readonly=1)
if result != 'OK':
raise Exception, message
typ, data = M.search(None, '(UNSEEN UNDELETED)')
for num in string.split(data[0]):
try:
f = M.fetch(num, '(BODY[HEADER.FIELDS (SUBJECT FROM)])')
m = rfc822.Message(msg(f[1][0][1]), 0)
subject = m['subject']
except KeyError:
f = M.fetch(num, '(BODY[HEADER.FIELDS (FROM)])')
m = rfc822.Message(msg(f[1][0][1]), 0)
subject = '(no subject)'
fromaddr = m.getaddr('from')
if fromaddr[0] == "": n = fromaddr[1]
else: n = fromaddr[0]
text = '%-20.20s %s' % (n, subject)
self.list.insert(END, text)
len = self.list.size()
if len > 0: self.list.see(len - 1)
except Exception, e:
self.list.delete(0, END)
print sys.exc_info()
self.list.insert(END, 'IMAP read error: ', e)
def _visit_story(self, dirname, name):
filepath = os.path.join(dirname,
name + '.' + Gyre.config.file_extension)
try:
s = os.stat(filepath)
except OSError:
return
story = Gyre.Entity(Gyre.config.protostory)
story.mtime = s.st_mtime
f = open(filepath)
headers = rfc822.Message(f)
for (key, val) in headers.items():
setattr(story, key.lower(), val.decode('utf-8'))
body = f.read().decode('utf-8')
f.close()
story.mtime = int(story.mtime)
categorystr = dirname[len(self.contentdir) + 1:]
if categorystr:
story.category = string.split(categorystr, '/')
else:
story.category = []
story.body = body
uid = list(story.category)
uid.append(name)
story.id = string.join(uid, '/')
Gyre.config.store.update(story)
def noheaders():
global _noheaders
if not _noheaders:
import rfc822
_noheaders = rfc822.Message(open('/dev/null', 'r'))
_noheaders.fp.close() # Recycle file descriptor
return _noheaders
def handle_request(self, request):
msg = rfc822.Message(StringIO('\n'.join(request.header)))
length = int(msg.get('Content-Length', '0'))
if length:
request.collector = xmlrpc_handler.collector(self, request)
else:
self.continue_request('', request)
def loadTemplateInfo(dir):
''' Attempt to load a Roundup template from the indicated directory.
Return None if there's no template, otherwise a template info
dictionary.
'''
ti = os.path.join(dir, 'TEMPLATE-INFO.txt')
if not os.path.exists(ti):
return None
if os.path.exists(os.path.join(dir, 'config.py')):
print _("WARNING: directory '%s'\n"
"\tcontains old-style template - ignored"
) % os.path.abspath(dir)
return None
# load up the template's information
f = open(ti)
try:
m = rfc822.Message(open(ti))
ti = {}
ti['name'] = m['name']
ti['description'] = m['description']
ti['intended-for'] = m['intended-for']
ti['path'] = dir
finally:
f.close()
return ti
def iter_mime_headers_and_bodies(wsgi_input, mime_boundary, read_chunk_size):
mime_documents_iter = iter_multipart_mime_documents(
wsgi_input, mime_boundary, read_chunk_size)
for file_like in mime_documents_iter:
hdrs = HeaderKeyDict(rfc822.Message(file_like, 0))
yield (hdrs, file_like)
def updateStore(self):
patchnames = map(string.strip,
os.popen(self.tlapath + ' logs').readlines())
stories = []
for patch in patchnames:
p = os.popen(self.tlapath + ' cat-log ' + patch)
msg = rfc822.Message(p)
full_tla_version = msg['Archive'] + '/' + msg['Revision']
body = p.read().strip()
body = cgi.escape(body)
body = body.replace('\n\n', '\n\n<p> ')
if not body:
body = '(no detail message)'
story = Gyre.Entity()
story.mtime = int(
time.mktime(
time.strptime(msg['Standard-date'],
'%Y-%m-%d %H:%M:%S GMT')))
story.subject = cgi.escape(msg['Summary'])
story.body = body
story.category = self.category
story.id = msg['Revision']
for header in ['Archive', 'Revision', 'Creator']:
setattr(story, header.lower(), msg[header])
story.full_tla_version = full_tla_version
story.full_tla_version_span = \
'<span class="tla-version">' + full_tla_version + '</span>'
stories.append(story)
for story in stories:
Gyre.config.store.update(story)
def parse_request(self):
"""
Parse a specific request and send error if any
Return False if error occurs; True if success
"""
try:
method, path, request_version = self.request_line.split()
except:
# Bad request syntax
self.send_error(400)
return False
if request_version not in self.supported_http_versions:
self.send_error(505)
return False
self.method = method
self.path = path
self.request_version = request_version
# Get optional headers
# Must use rfc822 or mimetools instead of email package, because
# `self.received` must be received using readline() method instead
# of read() which will block until the other end (client) closed the
# connection
self.headers = rfc822.Message(self.received, 0)
connection_type = self.headers.get('Connection', '')
if connection_type.lower() == 'close':
self.close_connection = True
elif (connection_type.lower() == 'keep-alive'
or self.request_version == 'HTTP/1.1'):
self.close_connection = False
return True
def specificTreatment(fo):
"""
Method contains the specific treatment of the file passed from NG/AMS.
fo: File object
Returns: (file_id, finalFileName, type);
The finalFileName is a string containing the name of the final
file without extension. type is the mime-type from the header.
"""
import rfc822, cgi, re
_EXT = '.msg'
filename = fo.name
uidTempl = re.compile(
"^[uU][iI][dD]://[aAbBcCzZxX][0-9,a-z,A-Z]+(/[xX][0-9,a-z,A-Z]+){2}(#\w{1,}|/\w{0,}){0,}$"
)
try:
message = rfc822.Message(fo)
type, tparams = cgi.parse_header(message["Content-Type"])
except Exception, e:
err = "Parsing of mime message failed: " + str(e)
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), _PLUGIN_ID, err])
raise Exception, errMsg
def next(self):
if not self.boxes:
return None
fn = self.boxes[0]
del self.boxes[0]
fp = open(fn)
return rfc822.Message(fp)
def next(self):
if not self.boxes:
return None
fn = self.boxes[0]
del self.boxes[0]
fp = open(os.path.join(self.dirname, fn))
return rfc822.Message(fp)
def eom(self):
if not self.fp: return Milter.ACCEPT
self.fp.seek(0)
msg = mime.message_from_file(self.fp)
msg.headerchange = self._headerChange
if not mime.defang(msg,self.tempname):
os.remove(self.tempname)
self.tempname = None # prevent re-removal
self.log("eom")
return Milter.ACCEPT # no suspicious attachments
self.log("Temp file:",self.tempname)
self.tempname = None # prevent removal of original message copy
# copy defanged message to a temp file
out = tempfile.TemporaryFile()
try:
msg.dump(out)
out.seek(0)
msg = rfc822.Message(out)
msg.rewindbody()
while 1:
buf = out.read(8192)
if len(buf) == 0: break
self.replacebody(buf) # feed modified message to sendmail
return Milter.ACCEPT # ACCEPT modified message
finally:
out.close()
return Milter.TEMPFAIL
def specificTreatment(fo):
"""
Method contains the specific treatment of the file passed from NG/AMS.
fo: File object
Returns: (file_id, finalFileName, type);
The finalFileName is a string containing the name of the final
file without extension. type is the mime-type from the header.
"""
import rfc822, cgi, re
_EXT = '.msg'
filename = fo.name
# uidTempl = re.compile("[xX][0-9,a-f]{3,}/[xX][0-9,a-f]{1,}$")
# This matches the old UID of type X0123456789abcdef/X01234567
uidTempl = re.compile(
"^[uU][iI][dD]:/(/[xX][0-9,a-f,A-F]+){3}(#\w{1,}){0,}$")
# This matches the new UID structure uid://X1/X2/X3#kdgflf
try:
message = rfc822.Message(fo)
type, tparams = cgi.parse_header(message["Content-Type"])
except Exception, e:
err = "Parsing of mime message failed: " + str(e)
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), _PLUGIN_ID, err])
raise Exception, errMsg
def get_messages(host, user, password):
"""Download all messages (e.g. pages) from the mailbox
Keyword arguments:
host -- IMAP server
user -- login connection information
password -- password connection information
See http://stackoverflow.com/questions/315362/properly-formatted-example-for-python-imap-email-access
"""
M = imaplib.IMAP4_SSL(host)
M.login(user, password)
M.select()
typ, data = M.search(None, 'UNSEEN')
messages = []
for num in data[0].split():
typ, data = M.fetch(num, '(RFC822)')
file = StringIO.StringIO(data[0][1])
message = rfc822.Message(file)
msgDate = time.mktime(rfc822.parsedate(message['date']))
messages.append(
create_event('IMAP', message['from'], message['to'],
message['subject'], int(msgDate)))
M.close()
M.logout()
log.info('Found %d unseen messages at %s', len(messages), host)
return messages
def _doCGI(conn, pid, stdin, stdout, stderr, stdindata):
text = _handleParentSide(pid, stdin, stdout, stderr, stdindata)
f = cStringIO.StringIO(text)
respp = rfc822.Message(f)
for k, v in respp.items():
conn.responseHeaders[k] = v
conn.write(f.read())
def do_POST(self):
"""
Syntax:
POST
Responses:
240 article posted ok
340 send article to be posted. End with <CR-LF>.<CR-LF>
440 posting not allowed
441 posting failed
"""
msg = rfc822.Message(StringIO.StringIO(''.join(self.article_lines)))
group_name = msg.getheader('Newsgroups')
# check the 'Newsgroups' header
backend = self._backend_from_group(group_name)
if (not backend # No backend matches Newsgroups: header
or not group_name # No Newsgroups: header
or not backend.group_exists(group_name)
): # Group not found in backend
self.send_response(ERR_POSTINGFAILED)
return
result = backend.do_POST(group_name, ''.join(self.article_lines),
self.client_address[0], self.auth_username)
if result == None:
self.send_response(ERR_POSTINGFAILED)
else:
self.send_response(STATUS_POSTSUCCESSFULL)
def open(self, id, rw='r'):
filename = self.id2filename(id)
if rw == 'w':
self.lock(id)
fp = open(filename)
msg = rfc822.Message(fp)
return self._objfactory(id, msg, rw)
def test(msg, results):
fp = StringIO()
fp.write(msg)
fp.seek(0)
m = rfc822.Message(fp)
i = 0
for n, a in m.getaddrlist('to') + m.getaddrlist('cc'):
if verbose:
print 'name:', repr(n), 'addr:', repr(a)
try:
mn, ma = results[i][0], results[i][1]
except IndexError:
print 'extra parsed address:', repr(n), repr(a)
continue
i = i + 1
if mn == n and ma == a:
if verbose:
print ' [matched]'
else:
if verbose:
print ' [no match]'
print 'not found:', repr(n), repr(a)
out = m.getdate('date')
if out:
if verbose:
print 'Date:', m.getheader('date')
if out == (1999, 1, 13, 23, 57, 35, 0, 0, 0):
if verbose:
print ' [matched]'
else:
if verbose:
print ' [no match]'
print 'Date conversion failed:', out
def __init__(self, id, rfcmessage, rw='r'):
self._headers = rfcmessage
dummy = self._headers.fp.readline()
if dummy != '\n':
self._headers.rewindbody()
data = rfc822.Message(rfcmessage.fp)
MdbObject.__init__(self, id, data, rw)
def process_reply_file(current, fname):
new_note = {}
reply = open(fname, "r")
msg = rfc822.Message(reply)
new_note['text'] = "%s\n%s" % (msg['From'], msg.fp.read())
new_note['timestamp'] = email.utils.parsedate_tz(msg['Date'])
current["notes"].append(new_note)
def get_message(self, group_name, id):
group = self._groupname2group(group_name)
filename = ''
try:
id = int(id)
try:
article = self.get_group_article_list(group)[id - 1]
filename = os.path.join(self.maildir_path, group, "cur",
article)
except IndexError:
return None
except ValueError:
# Treat non-numeric ID as Message-ID
try:
filename = self.cache.message_bymid(id.strip())['filename']
except TypeError:
# message_bymid() returned None
return None
try:
return rfc822.Message(open(filename))
except IOError:
return None
def load_documents(dirname):
dir_walker = os.walk(dirname)
all_documents = {}
for directory in dir_walker:
dirpath, dirnames, filenames = directory
if not len(filenames):
continue
category = dirpath.split(os.sep)[-1]
if not len(category):
continue
if category not in all_documents:
all_documents[category] = []
for fname in filenames:
full_fname = os.sep.join([dirpath, fname])
with open(full_fname) as f:
headers = rfc822.Message(f)
message = f.read().decode('ascii', errors='ignore')
all_documents[category].append((headers, message))
return all_documents
def select_letters_pop3(server, filters, quiet=False):
"""
Processes message queue according to specified filters.
Download's available message headers according to POP3 protocol.
Checks every header to meet all filter's criteria.
Tries to implement behavior similar to Python imaplib's mailbox.search() method.
Takes authenticated pop3 server instance and email filters.
Returns a tuple: list of message id's
and a filename_filter_applied = False or file filter value if it is applied.
"""
#retrieving headers of all server messages
msg_ids_list = []
resp, items, octets = server.list()
messagesInfo = server.list()[1]
print messagesInfo
for item in items:
msg_id, size = item.split()
response, lines, octets = server.top(msg_id, 3)
#converting to rfc822 compatible format
text = string.join(lines, "\n")
file = StringIO.StringIO(text)
message = rfc822.Message(file)
message_pass, filename_filter_applied = filter_message_pop3(
message, filters, quiet=quiet)
if message_pass:
msg_ids_list.append(msg_id)
if not quiet:
print "Will process message id's: "
print msg_ids_list
if filename_filter_applied:
print 'Filename search string value: ' + str(
filename_filter_applied)
return msg_ids_list, filename_filter_applied
def _processRequest(conn, sessionDict):
DEBUG(PYCGI, "pycgi processing request")
try:
env = _fix(conn.env, conn.uri)
except vfs.FileNotFoundException:
DEBUG(PYCGI, "file not found!")
return fourOhFourHandler(conn, sessionDict)
DEBUG(PYCGI, "file evidently exists")
oldpwd = os.getcwd()
os.environ.update(env)
# what to do with argv?
save_argv = sys.argv
sys.argv = [env['SCRIPT_NAME']]
saved = _swap_streams(conn)
try:
try:
(directory, file) = os.path.split(env['SCRIPT_FILENAME'])
os.chdir(directory)
DEBUG(PYCGI, "about to execfile %s" % file)
execfile(file)
except SystemExit:
pass
except:
logException()
raise
finally:
os.chdir(oldpwd)
sys.argv = save_argv
new = _swap_streams(conn, saved)
new[0].seek(0)
respp = rfc822.Message(new[0])
for k, v in respp.items():
conn.responseHeaders[k] = v
conn.write(new[0].read())
return conn.response()
def _list(self):
ret = []
(result, list) = self.conn.select(globals.imap_mailbox)
if result != "OK":
raise BackendException(list[0])
# Going to find all the archives which have remote_dir in the From
# address
# Search returns an error if you haven't selected an IMAP folder.
(result, list) = self.conn.search(None, 'FROM', self.remote_dir)
if result != "OK":
raise Exception(list[0])
if list[0] == '':
return ret
nums = list[0].strip().split(" ")
set = "%s:%s" % (nums[0], nums[-1])
(result, list) = self.conn.fetch(set, "(BODY[HEADER])")
if result != "OK":
raise Exception(list[0])
for msg in list:
if (len(msg) == 1):
continue
io = StringIO.StringIO(msg[1]) # pylint: disable=unsubscriptable-object
m = rfc822.Message(io)
subj = m.getheader("subject")
header_from = m.getheader("from")
# Catch messages with empty headers which cause an exception.
if (not (header_from is None)):
if (re.compile("^" + self.remote_dir + "$").match(header_from)):
ret.append(subj)
log.Info("IMAP LIST: %s %s" % (subj, header_from))
return ret
def message(f, delimiter = ''):
sts = 0
# Parse RFC822 header
m = rfc822.Message(f)
# Write unix header line
fullname, email = m.getaddr('From')
tt = m.getdate('Date')
if tt:
t = time.mktime(tt)
else:
sys.stderr.write(
'Unparseable date: %s\n' % `m.getheader('Date')`)
t = os.fstat(f.fileno())[stat.ST_MTIME]
print 'From', email, time.ctime(t)
# Copy RFC822 header
for line in m.headers:
print line,
print
# Copy body
while 1:
line = f.readline()
if line == delimiter:
break
if not line:
sys.stderr.write('Unexpected EOF in message\n')
sts = 1
break
if line[:5] == 'From ':
line = '>' + line
print line,
# Print trailing newline
print
return sts
def decode_chunked(self):
"""Decode the 'chunked' transfer coding."""
cl = 0
data = StringIO.StringIO()
while True:
line = self.rfile.readline().strip().split(";", 1)
chunk_size = int(line.pop(0), 16)
if chunk_size <= 0:
break
## if line: chunk_extension = line[0]
cl += chunk_size
data.write(self.rfile.read(chunk_size))
crlf = self.rfile.read(2)
if crlf != "\r\n":
self.simple_response("400 Bad Request",
"Bad chunked transfer coding "
"(expected '\\r\\n', got %r)" % crlf)
return
# Grab any trailer headers
headers = rfc822.Message(self.rfile, seekable=0)
self.environ.update(self.parse_headers(headers))
data.seek(0)
self.environ["wsgi.input"] = data
self.environ["CONTENT_LENGTH"] = str(cl) or ""
return True
class Mail:
SERVER = "pop.gmail.com"
USER = "******"
PASSWORD = "******"
logging.debug('connecting to ' + SERVER)
server = poplib.POP3_SSL(SERVER)
# login
logging.debug('logging in')
server.user(USER)
server.pass_(PASSWORD)
# list items on server
logging.debug('listing emails')
resp, items, octets = server.list()
# download the first message in the list
id, size = string.split(items[-1])
resp, text, octets = server.retr(id)
# convert list to Message object
text = string.join(text, "\n")
file = StringIO.StringIO(text)
message = rfc822.Message(file)
email = ''
# sorting mails
if (message['From'] == '*'):
email = (message['Subject'])
def _process_multipart_body(self, mimeinput, charset):
headers = StringIO()
lines = mimeinput.readpart()
for line in lines:
headers.write(line)
if line == '\r\n':
break
headers.seek(0)
headers = rfc822.Message(headers)
ctype, ctype_params = parse_header(headers.get('content-type', ''))
if ctype and 'charset' in ctype_params:
charset = ctype_params['charset']
cdisp, cdisp_params = parse_header(
headers.get('content-disposition', ''))
if not cdisp:
raise RequestError('expected Content-Disposition header')
name = cdisp_params.get('name')
filename = cdisp_params.get('filename')
if not (cdisp == 'form-data' and name):
raise RequestError('expected Content-Disposition: form-data'
'with a "name" parameter: got %r' %
headers.get('content-disposition', ''))
# FIXME: should really to handle Content-Transfer-Encoding and other
# MIME complexity here. See RFC2048 for the full horror story.
if filename:
# it might be large file upload so use a temporary file
upload = Upload(filename, ctype, charset)
upload.receive(lines)
_add_field_value(self.form, name, upload)
else:
value = _decode_string(''.join(lines), charset
or self.DEFAULT_CHARSET)
_add_field_value(self.form, name, value)
