def getHeaderAndContentsAsDictionaries(fd=sys.stdin, delimiter='\t'):
contents = fd.read()
contentsFileObj = cStringIO.cStringIO(contents)
dr = csv.DictReader(contentsFileObj, delimiter=delimiter)
rows = [row for row in dr]
fd2 = cStringIO.cStringIO(contents)
headerFields = fd2.readline().split(delimiter)
fd2.close()
return headerFields, rows
def test_defaults_os_x(self):
sys.platform = 'darwin'
reload(debops.config)
cfgparser = configparser.SafeConfigParser()
cfgparser.readfp(cStringIO(debops.config.DEFAULTS))
self.assertEqual(cfgparser.get('paths', 'data-home'),
'~/Library/Application Support/debops')
def run(self, tasks, name):
if self.status != RUNNING and self.todo:
self.status = RUNNING
else:
return
for parent in self.parents:
if tasks[parent].getTodo():
self.status = WAITING
return
elif tasks[parent].getStatus() == FAILURE \
or tasks[parent].getStatus() == ABORT \
or not self.validate(tasks):
self.status = ABORT
self.todo = False
if tasks[parent].getStatus() == FAILURE \
or tasks[parent].getStatus() == ABORT:
self.abort_reason = "Parent error : "+parent
else:
self.abort_reason = "Bad return code from parent"
return
if self.status == RUNNING:
child = {}
try:
print "START", name, time.strftime("%c")
(child["in"], child["out"], child["err"]) = os.popen3(self.command)
except Exception, (errno, strerror):
child["err"] = cStringIO.cStringIO("-"+str(errno)+" - "+strerror+"-")
self.status = FAILURE
self.todo = False
return
self.returncode = child["err"].read()
self.status = COMPLETION
self.todo = False
print "END", name, time.strftime("%c")
def test_read_numeric():
# make reader-like thing
str_io = cStringIO()
r = _make_readerlike(str_io)
# check simplest of tags
for base_dt, val, mdtype in (
('u2', 30, mio5p.miUINT16),
('i4', 1, mio5p.miINT32),
('i2', -1, mio5p.miINT16)):
for byte_code in ('<', '>'):
r.byte_order = byte_code
c_reader = m5u.VarReader5(r)
yield assert_equal, c_reader.little_endian, byte_code == '<'
yield assert_equal, c_reader.is_swapped, byte_code != boc.native_code
for sde_f in (False, True):
dt = np.dtype(base_dt).newbyteorder(byte_code)
a = _make_tag(dt, val, mdtype, sde_f)
a_str = a.tostring()
_write_stream(str_io, a_str)
el = c_reader.read_numeric()
yield assert_equal, el, val
# two sequential reads
_write_stream(str_io, a_str, a_str)
el = c_reader.read_numeric()
yield assert_equal, el, val
el = c_reader.read_numeric()
yield assert_equal, el, val
def test_read_stream():
tag = _make_tag('i4', 1, mio5p.miINT32, sde=True)
tag_str = tag.tostring()
str_io = cStringIO(tag_str)
st = streams.make_stream(str_io)
s = streams._read_into(st, tag.itemsize)
yield assert_equal, s, tag.tostring()
def test_zero_byte_string():
# Tests hack to allow chars of non-zero length, but 0 bytes
# make reader-like thing
str_io = cStringIO()
r = _make_readerlike(str_io, '<')
c_reader = m5u.VarReader5(r)
tag_dt = np.dtype([('mdtype', 'u4'), ('byte_count', 'u4')])
tag = np.zeros((1,), dtype=tag_dt)
tag['mdtype'] = mio5p.miINT8
tag['byte_count'] = 1
hdr = m5u.VarHeader5()
# Try when string is 1 length
hdr.set_dims([1,])
_write_stream(str_io, tag.tostring() + asbytes(' '))
str_io.seek(0)
val = c_reader.read_char(hdr)
assert_equal(val, u' ')
# Now when string has 0 bytes 1 length
tag['byte_count'] = 0
_write_stream(str_io, tag.tostring())
str_io.seek(0)
val = c_reader.read_char(hdr)
assert_equal(val, u' ')
# Now when string has 0 bytes 4 length
str_io.seek(0)
hdr.set_dims([4,])
val = c_reader.read_char(hdr)
assert_array_equal(val, [u' '] * 4)
def test_defaults_linux(self):
sys.platform = 'linux2'
reload(debops.config)
cfgparser = configparser.SafeConfigParser()
cfgparser.readfp(cStringIO(debops.config.DEFAULTS))
self.assertEqual(cfgparser.get('paths', 'data-home'),
'$XDG_DATA_HOME/debops')
def make_xpi(files):
file_obj = cStringIO()
zip_file = zipfile.ZipFile(file_obj, 'w')
for path, data in files.items():
zip_file.writestr(path, data)
zip_file.close()
file_obj.seek(0)
return file_obj
def test_2StringIO(self):
"""
Python 2's L{StringIO} and L{cStringIO} modules are both binary I/O.
"""
from cStringIO import StringIO as cStringIO
from StringIO import StringIO
self.assertEqual(ioType(StringIO()), bytes)
self.assertEqual(ioType(cStringIO()), bytes)
def test_2StringIO(self):
"""
Python 2's L{StringIO} and L{cStringIO} modules are both binary I/O.
"""
from cStringIO import StringIO as cStringIO
from StringIO import StringIO
self.assertEquals(ioType(StringIO()), bytes)
self.assertEquals(ioType(cStringIO()), bytes)
def make_xpi(files):
f = cStringIO()
z = ZipFile(f, 'w')
for path, data in files.items():
z.writestr(path, data)
z.close()
f.seek(0)
return f
def string_io(data=None): # cStringIO can't handle unicode
'''
Pass data through to stringIO module and return result
'''
try:
return cStringIO(bytes(data))
except UnicodeEncodeError:
return pyStringIO(data)
def test_defaults_os_x(self):
sys.platform = 'darwin'
reload(debops.config)
cfgparser = configparser.SafeConfigParser()
cfgparser.readfp(cStringIO(debops.config.DEFAULTS))
self.assertEqual(
cfgparser.get('paths', 'data-home'),
os.path.expanduser('~/Library/Application Support/debops'))
def test_defaults_windows_without_APPDATA(self):
sys.platform = 'win32'
unsetenv('APPDATA')
reload(debops.config)
cfgparser = configparser.SafeConfigParser()
cfgparser.readfp(cStringIO(debops.config.DEFAULTS))
self.assertEqual(cfgparser.get('paths', 'data-home'),
'~\\Application Data/debops')
def test_defaults_windows_with_APPDATA(self):
sys.platform = 'win32'
setenv('APPDATA', 'H:\\my\\own\\data')
reload(debops.config)
cfgparser = configparser.SafeConfigParser()
cfgparser.readfp(cStringIO(debops.config.DEFAULTS))
self.assertEqual(cfgparser.get('paths', 'data-home'),
'H:\\my\\own\\data/debops')
def test_defaults_windows_with_APPDATA(self):
sys.platform = 'win32'
setenv('APPDATA', 'H:\\my\\own\\data')
reload(debops.config)
cfgparser = configparser.SafeConfigParser()
cfgparser.readfp(cStringIO(debops.config.DEFAULTS))
self.assertEqual(cfgparser.get('paths', 'data-home'),
'H:\\my\\own\\data/debops')
def test_defaults_windows_without_APPDATA(self):
sys.platform = 'win32'
unsetenv('APPDATA')
reload(debops.config)
cfgparser = configparser.SafeConfigParser()
cfgparser.readfp(cStringIO(debops.config.DEFAULTS))
self.assertEqual(cfgparser.get('paths', 'data-home'),
'~\\Application Data/debops')
def make_xpi(files):
f = cStringIO()
z = ZipFile(f, 'w')
for path, data in files.items():
z.writestr(path, data)
z.close()
f.seek(0)
return f
def process_response(self, request, response):
if settings.DEBUG and 'prof' in request.GET:
self.profiler.create_stats()
io = cStringIO()
stats = pstats.Stats(self.profiler, stream=io)
stats.strip_dirs().sort_stats(request.GET.get('sort', 'time'))
stats.print_stats(int(request.GET.get('count', 10000)))
response.content = '<pre>%s</pre>' % io.getvalue()
return response
def process_response(self, request, response):
if settings.DEBUG and 'prof' in request.GET:
self.profiler.create_stats()
io = cStringIO()
stats = pstats.Stats(self.profiler, stream=io)
stats.strip_dirs().sort_stats(request.GET.get('sort', 'time'))
stats.print_stats(int(request.GET.get('count', 10000)))
response.content = '<pre>%s</pre>' % io.getvalue()
return response
def setup_test():
global streams
val = b'a\x00string'
fs = TemporaryFile()
fs.write(val)
fs.seek(0)
gs = BytesIO(val)
cs = cStringIO(val)
streams = [fs, gs, cs]
def __init__(self, array_reader, byte_count):
super(Mat5ZArrayReader, self).__init__(
cStringIO(zlib.decompress(
array_reader.mat_stream.read(byte_count))),
array_reader.dtypes,
array_reader.processor_func,
array_reader.codecs,
array_reader.class_dtypes,
array_reader.struct_as_record)
def prepare_transform(self, data, arguments=None):
"""
The method takes some data in one of the input formats and returns
a TransformResult with data in the output format.
"""
if arguments is not None:
infile_data_suffix = arguments.get('infile_data_suffix', False)
del arguments['infile_data_suffix']
result = TransformResult(None)
tmpfilepath = None
try:
tmpdirpath = tempfile.mkdtemp()
tmpfilepath = self.initialize_tmpfile(data, directory=tmpdirpath)
if infile_data_suffix:
primaryname = os.path.basename(tmpfilepath) + infile_data_suffix
commandline = 'cd "%s" && %s %s' % (
tmpdirpath, self.command, self.args)
arguments['infile'] = tmpfilepath
commandline = commandline % arguments
if os.name=='posix':
# TODO: tbenita suggests to remove 1>/dev/null as some commands
# return result in flow. Maybe turn this into another subobject
# commandline = commandline + ' 2>error_log'
commandline = commandline + ' 2>error_log 1>/dev/null'
os.system(commandline)
for tmpfile in os.listdir(tmpdirpath):
tmp = os.path.join(tmpdirpath, tmpfile)
# Exclude the original file and the error_log from the result
if tmp == tmpfilepath:
continue
fd = file(tmp, 'rb')
# Should we use the infile as the primary output?
if infile_data_suffix and primaryname == tmpfile:
result.data = StringIO()
result.data.writelines(fd)
result.data.seek(0)
elif tmp.endswith('error_log'):
result.errors = fd.read()
else:
sub = cStringIO()
sub.writelines(fd)
sub.seek(0)
result.subobjects[tmpfile] = sub
fd.close()
os.unlink(tmp)
finally:
if tmpfilepath is not None and os.path.isdir(tmpdirpath):
shutil.rmtree(tmpdirpath)
return result
def _update_caps(self, caps):
# Rewrite the moov in the streamheaders buffer
raw_headers = cStringIO()
iso.write_atoms([self._moov], raw_headers)
self._streamheaders = gst.Buffer(raw_headers.getvalue())
self._streamheaders.flag_set(gst.BUFFER_FLAG_IN_CAPS)
s = caps[0].copy()
s.set_value('streamheader', (self._streamheaders, ))
self._outputcaps = gst.Caps(s)
self._streamheaders.set_caps(self._outputcaps)
def text2wave(text): process = subprocess.Popen(_text2wave, stdin=subprocess.PIPE, stdout=subprocess.PIPE) try: process.stdin.write(text + os.linesep) except TypeError: process.stdin.write(bytes(text + os.linesep, "utf-8")) (output, err) = process.communicate() exitCode = process.wait() wavf = cStringIO(output) return wave.open(wavf, "rb")
def setup():
val = b'a\x00string'
global fs, gs, cs, fname
fd, fname = mkstemp()
fs = os.fdopen(fd, 'wb')
fs.write(val)
fs.close()
fs = open(fname, 'rb')
gs = BytesIO(val)
cs = cStringIO(val)
def setup():
val = b'a\x00string'
global fs, gs, cs, fname
fd, fname = mkstemp()
fs = os.fdopen(fd, 'wb')
fs.write(val)
fs.close()
fs = open(fname, 'rb')
gs = BytesIO(val)
cs = cStringIO(val)
def setup_test_file():
val = b'a\x00string'
fd, fname = mkstemp()
with os.fdopen(fd, 'wb') as fs:
fs.write(val)
with open(fname, 'rb') as fs:
gs = BytesIO(val)
cs = cStringIO(val)
yield fs, gs, cs
os.unlink(fname)
def setup_test_file():
val = b'a\x00string'
fd, fname = mkstemp()
with os.fdopen(fd, 'wb') as fs:
fs.write(val)
with open(fname, 'rb') as fs:
gs = BytesIO(val)
cs = cStringIO(val)
yield fs, gs, cs
os.unlink(fname)
def test_read_numeric_writeable():
# make reader-like thing
str_io = cStringIO()
r = _make_readerlike(str_io, '<')
c_reader = m5u.VarReader5(r)
dt = np.dtype('<u2')
a = _make_tag(dt, 30, mio5p.miUINT16, 0)
a_str = a.tostring()
_write_stream(str_io, a_str)
el = c_reader.read_numeric()
yield assert_true, el.flags.writeable
def test_read_numeric_writeable():
# make reader-like thing
str_io = cStringIO()
r = _make_readerlike(str_io, '<')
c_reader = m5u.VarReader5(r)
dt = np.dtype('<u2')
a = _make_tag(dt, 30, mio5p.miUINT16, 0)
a_str = a.tostring()
_write_stream(str_io, a_str)
el = c_reader.read_numeric()
yield assert_true, el.flags.writeable
def test_write_to(self):
b = self.nested_buffer()
fp = cStringIO()
b.write_to(fp, up_to = b.world, close=True)
self.assertEqual("Hallo", fp.getvalue())
def r(buff):
buff.write(" ")
# The first buffer may not be written to any more.
self.assertRaises(IllegalFunctionCall, r, b.hello)
def test_read_numeric_writeable():
# make reader-like thing
str_io = cStringIO()
r = _make_readerlike()
r.mat_stream = str_io
r.byte_order = '<'
r.dtypes = miob.convert_dtypes(mio5.mdtypes_template, '<')
c_reader = m5u.VarReader5(r)
dt = np.dtype('<u2')
a = _make_tag(dt, 30, mio5.miUINT16, 0)
a_str = a.tostring()
_write_stream(str_io, a_str)
el = c_reader.read_numeric()
yield assert_true, el.flags.writeable
def save_mail_to_librarian(raw_mail, restricted=False):
"""Save the message to the librarian.
It can be referenced from errors, and also accessed by code that needs to
get back the exact original form.
"""
# File the raw_mail in the Librarian. We generate a filename to avoid
# people guessing the URL. We don't want URLs to private bug messages to
# be guessable for example.
file_name = str(uuid1()) + '.txt'
file_alias = getUtility(ILibraryFileAliasSet).create(
file_name, len(raw_mail), cStringIO(raw_mail), 'message/rfc822',
restricted=restricted)
return file_alias
def run(self, tasks, name):
if self.status != RUNNING and self.todo:
self.status = RUNNING
else:
return
for parent in self.parents:
if tasks[parent].getTodo():
self.status = WAITING
return
elif tasks[parent].getStatus() == FAILURE \
or tasks[parent].getStatus() == ABORT \
or not self.validate(tasks):
self.status = ABORT
self.todo = False
if tasks[parent].getStatus() == FAILURE \
or tasks[parent].getStatus() == ABORT:
self.abort_reason = "Parent error : " + parent
else:
self.abort_reason = "Bad return code from parent"
return
if self.status == RUNNING:
child = {}
try:
print "START", name, time.strftime("%c")
(child["in"], child["out"],
child["err"]) = os.popen3(self.command)
except Exception, (errno, strerror):
child["err"] = cStringIO.cStringIO("-" + str(errno) + " - " +
strerror + "-")
self.status = FAILURE
self.todo = False
return
self.returncode = child["err"].read()
self.status = COMPLETION
self.todo = False
print "END", name, time.strftime("%c")
def _parse_headers(self, buf, tid):
# Parse buffer and get the moov atom
f = cStringIO(buf.data)
al = list(atoms.read_atoms(f))
ad = atoms.atoms_dict(al)
try:
moov = iso.select_atoms(ad, ('moov', 1, 1))[0]
except Exception:
self.error("Could not parse moov")
m = messages.Error(T_(N_(
"First buffer cannot be parsed as a moov. "\
"The required mp4seek version is 1.0-6")))
self.addMessage(m)
return False
# Modify the track id and update the streamheaders buffer
trak = moov.trak[0]
trak.tkhd.id = tid
if self._moov is None:
self._moov = moov
else:
self._moov.trak.append(trak)
self._update_caps(buf.caps)
return True
def StringIO(s=None): # cStringIO can't handle unicode
try:
return cStringIO(bytes(s))
except UnicodeEncodeError:
return pyStringIO(s)
def _verifyDkimOrigin(signed_message):
"""Find a From or Sender address for which there's a DKIM signature.
:returns: A string email address for the trusted sender, if there is one,
otherwise None.
:param signed_message: ISignedMessage
"""
log = logging.getLogger('mail-authenticate-dkim')
log.setLevel(logging.DEBUG)
if getFeatureFlag('mail.dkim_authentication.disabled'):
log.info('dkim authentication feature disabled')
return None
# uncomment this for easier test debugging
# log.addHandler(logging.FileHandler('/tmp/dkim.log'))
dkim_log = cStringIO()
log.info(
'Attempting DKIM authentication of message id=%r from=%r sender=%r' %
(signed_message['Message-ID'], signed_message['From'],
signed_message['Sender']))
signing_details = []
dkim_result = False
try:
dkim_result = dkim.verify(signed_message.parsed_string,
dkim_log,
details=signing_details)
except dkim.DKIMException as e:
log.warning('DKIM error: %r' % (e, ))
except dns.resolver.NXDOMAIN as e:
# This can easily happen just through bad input data, ie claiming to
# be signed by a domain with no visible key of that name. It's not an
# operational error.
log.info('DNS exception: %r' % (e, ))
except dns.exception.DNSException as e:
# many of them have lame messages, thus %r
log.warning('DNS exception: %r' % (e, ))
except Exception as e:
# DKIM leaks some errors when it gets bad input, as in bug 881237. We
# don't generally want them to cause the mail to be dropped entirely
# though. It probably is reasonable to treat them as potential
# operational errors, at least until they're handled properly, by
# making pydkim itself more defensive.
log.warning(
'unexpected error in DKIM verification, treating as unsigned: %r' %
(e, ))
log.info('DKIM verification result: trusted=%s' % (dkim_result, ))
log.debug('DKIM debug log: %s' % (dkim_log.getvalue(), ))
if not dkim_result:
return None
# in addition to the dkim signature being valid, we have to check that it
# was actually signed by the user's domain.
if len(signing_details) != 1:
log.info('expected exactly one DKIM details record: %r' %
(signing_details, ))
return None
signing_domain = signing_details[0]['d']
if not _isDkimDomainTrusted(signing_domain):
log.info("valid DKIM signature from untrusted domain %s" %
(signing_domain, ))
return None
for origin in ['From', 'Sender']:
if signed_message[origin] is None:
continue
name, addr = parseaddr(signed_message[origin])
try:
origin_domain = addr.split('@')[1]
except IndexError:
log.warning("couldn't extract domain from address %r",
signed_message[origin])
if signing_domain == origin_domain:
log.info("DKIM signing domain %s matches %s address %r",
signing_domain, origin, addr)
return addr
else:
log.info("DKIM signing domain %s doesn't match message origin; "
"disregarding signature" % (signing_domain))
return None
def getImage():
stream = cStringIO.cStringIO(data)
return wx.ImageFromStream( stream )
def fromEmail(self,
email_message,
owner=None,
filealias=None,
parsed_message=None,
create_missing_persons=False,
fallback_parent=None,
date_created=None,
restricted=False):
"""See IMessageSet.fromEmail."""
# It does not make sense to handle Unicode strings, as email
# messages may contain chunks encoded in differing character sets.
# Passing Unicode in here indicates a bug.
if not zisinstance(email_message, str):
raise TypeError('email_message must be a normal string. Got: %r' %
email_message)
# Parse the raw message into an email.message.Message instance,
# if we haven't been given one already.
if parsed_message is None:
parsed_message = email.message_from_string(email_message)
# We could easily generate a default, but a missing message-id
# almost certainly means a developer is using this method when
# they shouldn't (by creating emails by hand and passing them here),
# which is broken because they will almost certainly have Unicode
# errors.
rfc822msgid = parsed_message.get('message-id')
if not rfc822msgid:
raise InvalidEmailMessage('Missing Message-Id')
# Over-long messages are checked for at the handle_on_message level.
# If it's a restricted mail (IE: for a private bug), or it hasn't been
# uploaded, do so now.
from lp.services.mail.helpers import save_mail_to_librarian
if restricted or filealias is None:
raw_email_message = save_mail_to_librarian(email_message,
restricted=restricted)
else:
raw_email_message = filealias
# Find the message subject
subject = self._decode_header(parsed_message.get('subject', ''))
subject = subject.strip()
if owner is None:
# Try and determine the owner. We raise a NotFoundError
# if the sender does not exist, unless we were asked to
# create_missing_persons.
person_set = getUtility(IPersonSet)
from_hdr = self._decode_header(parsed_message.get('from',
'')).strip()
replyto_hdr = self._decode_header(
parsed_message.get('reply-to', '')).strip()
from_addrs = [from_hdr, replyto_hdr]
from_addrs = [parseaddr(addr) for addr in from_addrs if addr]
if len(from_addrs) == 0:
raise InvalidEmailMessage('No From: or Reply-To: header')
for from_addr in from_addrs:
owner = person_set.getByEmail(from_addr[1].lower().strip())
if owner is not None:
break
if owner is None:
if not create_missing_persons:
raise UnknownSender(from_addrs[0][1])
# autocreate a person
sendername = ensure_unicode(from_addrs[0][0].strip())
senderemail = from_addrs[0][1].lower().strip()
# XXX: Guilherme Salgado 2006-08-31 bug=62344:
# It's hard to define what rationale to use here, and to
# make things worst, it's almost impossible to provide a
# meaningful comment having only the email message.
owner = person_set.ensurePerson(
senderemail, sendername,
PersonCreationRationale.FROMEMAILMESSAGE)
if owner is None:
raise UnknownSender(senderemail)
# Get the parent of the message, if available in the db. We'll
# go through all the message's parents until we find one that's
# in the db.
parent = None
for parent_msgid in reversed(get_parent_msgids(parsed_message)):
try:
# we assume it's the first matching message
parent = self.get(parent_msgid)[0]
break
except NotFoundError:
pass
else:
parent = fallback_parent
# Figure out the date of the message.
if date_created is not None:
datecreated = date_created
else:
datecreated = utcdatetime_from_field(parsed_message['date'])
# Make sure we don't create an email with a datecreated in the
# distant past or future.
now = datetime.now(pytz.timezone('UTC'))
thedistantpast = datetime(1990, 1, 1, tzinfo=pytz.timezone('UTC'))
if datecreated < thedistantpast or datecreated > now:
datecreated = UTC_NOW
message = Message(subject=subject,
owner=owner,
rfc822msgid=rfc822msgid,
parent=parent,
raw=raw_email_message,
datecreated=datecreated)
sequence = 1
# Don't store the preamble or epilogue -- they are only there
# to give hints to non-MIME aware clients
#
# Determine the encoding to use for non-multipart messages, and the
# preamble and epilogue of multipart messages. We default to
# iso-8859-1 as it seems fairly harmless to cope with old, broken
# mail clients (The RFCs state US-ASCII as the default character
# set).
# default_charset = (parsed_message.get_content_charset() or
# 'iso-8859-1')
#
# XXX: kiko 2005-09-23: Is default_charset only useful here?
#
# if getattr(parsed_message, 'preamble', None):
# # We strip a leading and trailing newline - the email parser
# # seems to arbitrarily add them :-/
# preamble = parsed_message.preamble.decode(
# default_charset, 'replace')
# if preamble.strip():
# if preamble[0] == '\n':
# preamble = preamble[1:]
# if preamble[-1] == '\n':
# preamble = preamble[:-1]
# MessageChunk(
# message=message, sequence=sequence, content=preamble
# )
# sequence += 1
for part in parsed_message.walk():
mime_type = part.get_content_type()
# Skip the multipart section that walk gives us. This part
# is the entire message.
if part.is_multipart():
continue
# Decode the content of this part.
content = part.get_payload(decode=True)
# Store the part as a MessageChunk
#
# We want only the content type text/plain as "main content".
# Exceptions to this rule:
# - if the content disposition header explicitly says that
# this part is an attachment, text/plain content is stored
# as a blob,
# - if the content-disposition header provides a filename,
# text/plain content is stored as a blob.
content_disposition = part.get('Content-disposition', '').lower()
no_attachment = not content_disposition.startswith('attachment')
if (mime_type == 'text/plain' and no_attachment
and part.get_filename() is None):
# Get the charset for the message part. If one isn't
# specified, default to latin-1 to prevent
# UnicodeDecodeErrors.
charset = part.get_content_charset()
if charset is None or str(charset).lower() == 'x-unknown':
charset = 'latin-1'
content = self.decode(content, charset)
if content.strip():
MessageChunk(message=message,
sequence=sequence,
content=content)
sequence += 1
else:
filename = part.get_filename() or 'unnamed'
# Strip off any path information.
filename = os.path.basename(filename)
# Note we use the Content-Type header instead of
# part.get_content_type() here to ensure we keep
# parameters as sent. If Content-Type is None we default
# to application/octet-stream.
if part['content-type'] is None:
content_type = 'application/octet-stream'
else:
content_type = part['content-type']
if len(content) > 0:
blob = getUtility(ILibraryFileAliasSet).create(
name=filename,
size=len(content),
file=cStringIO(content),
contentType=content_type,
restricted=restricted)
MessageChunk(message=message, sequence=sequence, blob=blob)
sequence += 1
# Don't store the epilogue
# if getattr(parsed_message, 'epilogue', None):
# epilogue = parsed_message.epilogue.decode(
# default_charset, 'replace')
# if epilogue.strip():
# if epilogue[0] == '\n':
# epilogue = epilogue[1:]
# if epilogue[-1] == '\n':
# epilogue = epilogue[:-1]
# MessageChunk(
# message=message, sequence=sequence, content=epilogue
# )
# XXX 2008-05-27 jamesh:
# Ensure that BugMessages get flushed in same order as they
# are created.
Store.of(message).flush()
return message
def fromEmail(self, email_message, owner=None, filealias=None,
parsed_message=None, create_missing_persons=False,
fallback_parent=None, date_created=None, restricted=False):
"""See IMessageSet.fromEmail."""
# It does not make sense to handle Unicode strings, as email
# messages may contain chunks encoded in differing character sets.
# Passing Unicode in here indicates a bug.
if not zisinstance(email_message, str):
raise TypeError(
'email_message must be a normal string. Got: %r'
% email_message)
# Parse the raw message into an email.Message.Message instance,
# if we haven't been given one already.
if parsed_message is None:
parsed_message = email.message_from_string(email_message)
# We could easily generate a default, but a missing message-id
# almost certainly means a developer is using this method when
# they shouldn't (by creating emails by hand and passing them here),
# which is broken because they will almost certainly have Unicode
# errors.
rfc822msgid = parsed_message.get('message-id')
if not rfc822msgid:
raise InvalidEmailMessage('Missing Message-Id')
# Over-long messages are checked for at the handle_on_message level.
# If it's a restricted mail (IE: for a private bug), or it hasn't been
# uploaded, do so now.
from lp.services.mail.helpers import save_mail_to_librarian
if restricted or filealias is None:
raw_email_message = save_mail_to_librarian(
email_message, restricted=restricted)
else:
raw_email_message = filealias
# Find the message subject
subject = self._decode_header(parsed_message.get('subject', ''))
subject = subject.strip()
if owner is None:
# Try and determine the owner. We raise a NotFoundError
# if the sender does not exist, unless we were asked to
# create_missing_persons.
person_set = getUtility(IPersonSet)
from_hdr = self._decode_header(
parsed_message.get('from', '')).strip()
replyto_hdr = self._decode_header(
parsed_message.get('reply-to', '')).strip()
from_addrs = [from_hdr, replyto_hdr]
from_addrs = [parseaddr(addr) for addr in from_addrs if addr]
if len(from_addrs) == 0:
raise InvalidEmailMessage('No From: or Reply-To: header')
for from_addr in from_addrs:
owner = person_set.getByEmail(from_addr[1].lower().strip())
if owner is not None:
break
if owner is None:
if not create_missing_persons:
raise UnknownSender(from_addrs[0][1])
# autocreate a person
sendername = ensure_unicode(from_addrs[0][0].strip())
senderemail = from_addrs[0][1].lower().strip()
# XXX: Guilherme Salgado 2006-08-31 bug=62344:
# It's hard to define what rationale to use here, and to
# make things worst, it's almost impossible to provide a
# meaningful comment having only the email message.
owner = person_set.ensurePerson(
senderemail, sendername,
PersonCreationRationale.FROMEMAILMESSAGE)
if owner is None:
raise UnknownSender(senderemail)
# Get the parent of the message, if available in the db. We'll
# go through all the message's parents until we find one that's
# in the db.
parent = None
for parent_msgid in reversed(get_parent_msgids(parsed_message)):
try:
# we assume it's the first matching message
parent = self.get(parent_msgid)[0]
break
except NotFoundError:
pass
else:
parent = fallback_parent
# Figure out the date of the message.
if date_created is not None:
datecreated = date_created
else:
datecreated = utcdatetime_from_field(parsed_message['date'])
# Make sure we don't create an email with a datecreated in the
# distant past or future.
now = datetime.now(pytz.timezone('UTC'))
thedistantpast = datetime(1990, 1, 1, tzinfo=pytz.timezone('UTC'))
if datecreated < thedistantpast or datecreated > now:
datecreated = UTC_NOW
message = Message(subject=subject, owner=owner,
rfc822msgid=rfc822msgid, parent=parent,
raw=raw_email_message, datecreated=datecreated)
sequence = 1
# Don't store the preamble or epilogue -- they are only there
# to give hints to non-MIME aware clients
#
# Determine the encoding to use for non-multipart messages, and the
# preamble and epilogue of multipart messages. We default to
# iso-8859-1 as it seems fairly harmless to cope with old, broken
# mail clients (The RFCs state US-ASCII as the default character
# set).
# default_charset = (parsed_message.get_content_charset() or
# 'iso-8859-1')
#
# XXX: kiko 2005-09-23: Is default_charset only useful here?
#
# if getattr(parsed_message, 'preamble', None):
# # We strip a leading and trailing newline - the email parser
# # seems to arbitrarily add them :-/
# preamble = parsed_message.preamble.decode(
# default_charset, 'replace')
# if preamble.strip():
# if preamble[0] == '\n':
# preamble = preamble[1:]
# if preamble[-1] == '\n':
# preamble = preamble[:-1]
# MessageChunk(
# message=message, sequence=sequence, content=preamble
# )
# sequence += 1
for part in parsed_message.walk():
mime_type = part.get_content_type()
# Skip the multipart section that walk gives us. This part
# is the entire message.
if part.is_multipart():
continue
# Decode the content of this part.
content = part.get_payload(decode=True)
# Store the part as a MessageChunk
#
# We want only the content type text/plain as "main content".
# Exceptions to this rule:
# - if the content disposition header explicitly says that
# this part is an attachment, text/plain content is stored
# as a blob,
# - if the content-disposition header provides a filename,
# text/plain content is stored as a blob.
content_disposition = part.get('Content-disposition', '').lower()
no_attachment = not content_disposition.startswith('attachment')
if (mime_type == 'text/plain' and no_attachment
and part.get_filename() is None):
# Get the charset for the message part. If one isn't
# specified, default to latin-1 to prevent
# UnicodeDecodeErrors.
charset = part.get_content_charset()
if charset is None or str(charset).lower() == 'x-unknown':
charset = 'latin-1'
content = self.decode(content, charset)
if content.strip():
MessageChunk(
message=message, sequence=sequence, content=content)
sequence += 1
else:
filename = part.get_filename() or 'unnamed'
# Strip off any path information.
filename = os.path.basename(filename)
# Note we use the Content-Type header instead of
# part.get_content_type() here to ensure we keep
# parameters as sent. If Content-Type is None we default
# to application/octet-stream.
if part['content-type'] is None:
content_type = 'application/octet-stream'
else:
content_type = part['content-type']
if len(content) > 0:
blob = getUtility(ILibraryFileAliasSet).create(
name=filename, size=len(content),
file=cStringIO(content), contentType=content_type,
restricted=restricted)
MessageChunk(message=message, sequence=sequence, blob=blob)
sequence += 1
# Don't store the epilogue
# if getattr(parsed_message, 'epilogue', None):
# epilogue = parsed_message.epilogue.decode(
# default_charset, 'replace')
# if epilogue.strip():
# if epilogue[0] == '\n':
# epilogue = epilogue[1:]
# if epilogue[-1] == '\n':
# epilogue = epilogue[:-1]
# MessageChunk(
# message=message, sequence=sequence, content=epilogue
# )
# XXX 2008-05-27 jamesh:
# Ensure that BugMessages get flushed in same order as they
# are created.
Store.of(message).flush()
return message
def convert(images, dpi=None, pagesize=(None, None), title=None, author=None,
creator=None, producer=None, creationdate=None, moddate=None,
subject=None, keywords=None, colorspace=None, nodate=False,
verbose=False):
pdf = pdfdoc(3, title, author, creator, producer, creationdate,
moddate, subject, keywords, nodate)
for imfilename in images:
debug_out("Reading %s"%imfilename, verbose)
try:
rawdata = imfilename.read()
except AttributeError:
with open(imfilename, "rb") as im:
rawdata = im.read()
im = cStringIO(rawdata)
try:
imgdata = Image.open(im)
except IOError as e:
# test if it is a jpeg2000 image
if rawdata[:12] != "\x00\x00\x00\x0C\x6A\x50\x20\x20\x0D\x0A\x87\x0A":
error_out("cannot read input image (not jpeg2000)")
error_out("PIL: %s"%e)
exit(1)
# image is jpeg2000
width, height, ics = parsejp2(rawdata)
imgformat = "JPEG2000"
if dpi:
ndpi = dpi, dpi
debug_out("input dpi (forced) = %d x %d"%ndpi, verbose)
else:
# TODO: read real dpi from input jpeg2000 image
ndpi = (96, 96)
debug_out("input dpi = %d x %d"%ndpi, verbose)
if colorspace:
color = colorspace
debug_out("input colorspace (forced) = %s"%(ics))
else:
color = ics
debug_out("input colorspace = %s"%(ics), verbose)
else:
width, height = imgdata.size
imgformat = imgdata.format
if dpi:
ndpi = dpi, dpi
debug_out("input dpi (forced) = %d x %d"%ndpi, verbose)
else:
ndpi = imgdata.info.get("dpi", (96, 96))
# in python3, the returned dpi value for some tiff images will
# not be an integer but a float. To make the behaviour of
# img2pdf the same between python2 and python3, we convert that
# float into an integer by rounding
# search online for the 72.009 dpi problem for more info
ndpi = (int(round(ndpi[0])),int(round(ndpi[1])))
debug_out("input dpi = %d x %d"%ndpi, verbose)
if colorspace:
color = colorspace
debug_out("input colorspace (forced) = %s"%(color), verbose)
else:
color = imgdata.mode
if color == "CMYK" and imgformat == "JPEG":
# Adobe inverts CMYK JPEGs for some reason, and others
# have followed suit as well. Some software assumes the
# JPEG is inverted if the Adobe tag (APP14), while other
# software assumes all CMYK JPEGs are inverted. I don't
# have enough experience with these to know which is
# better for images currently in the wild, so I'm going
# with the first approach for now.
if "adobe" in imgdata.info:
color = "CMYK;I"
debug_out("input colorspace = %s"%(color), verbose)
debug_out("width x height = %d x %d"%(width,height), verbose)
debug_out("imgformat = %s"%imgformat, verbose)
# depending on the input format, determine whether to pass the raw
# image or the zlib compressed color information
if imgformat is "JPEG" or imgformat is "JPEG2000":
if color == '1':
error_out("jpeg can't be monochrome")
exit(1)
imgdata = rawdata
else:
# because we do not support /CCITTFaxDecode
if color == '1':
debug_out("Converting colorspace 1 to L", verbose)
imgdata = imgdata.convert('L')
color = 'L'
elif color in ("RGB", "L", "CMYK", "CMYK;I"):
debug_out("Colorspace is OK: %s"%color, verbose)
else:
debug_out("Converting colorspace %s to RGB"%color, verbose)
imgdata = imgdata.convert('RGB')
color = imgdata.mode
img = imgdata.tobytes()
# the python-pil version 2.3.0-1ubuntu3 in Ubuntu does not have the close() method
try:
imgdata.close()
except AttributeError:
pass
imgdata = zlib.compress(img)
im.close()
# pdf units = 1/72 inch
if not pagesize[0] and not pagesize[1]:
pdf_x, pdf_y = 72.0*width/float(ndpi[0]), 72.0*height/float(ndpi[1])
elif not pagesize[1]:
pdf_x, pdf_y = pagesize[0], pagesize[0]*height/float(width)
elif not pagesize[0]:
pdf_x, pdf_y = pagesize[1]*width/float(height), pagesize[1]
else:
pdf_x = pagesize[0]
pdf_y = pagesize[1]
pdf.addimage(color, width, height, imgformat, imgdata, pdf_x, pdf_y)
return pdf.tostring()
def from_string(cls, s):
"""
Create a dsc_document from string s.
"""
return cls.from_file(cStringIO(s))
def test_render_output_stream_utf8(self):
xml = XML('<li>Über uns</li>')
strio = cStringIO()
self.assertEqual(None, xml.render(out=strio))
self.assertEqual('<li>Über uns</li>', strio.getvalue())
def test_render_output_stream_utf8(self):
xml = XML('<li>Über uns</li>')
strio = cStringIO()
self.assertEqual(None, xml.render(out=strio))
self.assertEqual('<li>Über uns</li>', strio.getvalue())
def test_stringio_bytes(self):
"""Verify that a Grammar can be created from file-like objects other than Python's standard 'file' object"""
Grammar(cStringIO(b"start: a+ b a+? 'b' a*; b: 'b'; a: 'a';"))
def _verifyDkimOrigin(signed_message):
"""Find a From or Sender address for which there's a DKIM signature.
:returns: A string email address for the trusted sender, if there is one,
otherwise None.
:param signed_message: ISignedMessage
"""
log = logging.getLogger('mail-authenticate-dkim')
log.setLevel(logging.DEBUG)
if getFeatureFlag('mail.dkim_authentication.disabled'):
log.info('dkim authentication feature disabled')
return None
# uncomment this for easier test debugging
# log.addHandler(logging.FileHandler('/tmp/dkim.log'))
dkim_log = cStringIO()
log.info(
'Attempting DKIM authentication of message id=%r from=%r sender=%r'
% (signed_message['Message-ID'],
signed_message['From'],
signed_message['Sender']))
signing_details = []
dkim_result = False
try:
dkim_result = dkim.verify(
signed_message.parsed_string, dkim_log, details=signing_details)
except dkim.DKIMException as e:
log.warning('DKIM error: %r' % (e,))
except dns.resolver.NXDOMAIN as e:
# This can easily happen just through bad input data, ie claiming to
# be signed by a domain with no visible key of that name. It's not an
# operational error.
log.info('DNS exception: %r' % (e,))
except dns.exception.DNSException as e:
# many of them have lame messages, thus %r
log.warning('DNS exception: %r' % (e,))
except Exception as e:
# DKIM leaks some errors when it gets bad input, as in bug 881237. We
# don't generally want them to cause the mail to be dropped entirely
# though. It probably is reasonable to treat them as potential
# operational errors, at least until they're handled properly, by
# making pydkim itself more defensive.
log.warning(
'unexpected error in DKIM verification, treating as unsigned: %r'
% (e,))
log.info('DKIM verification result: trusted=%s' % (dkim_result,))
log.debug('DKIM debug log: %s' % (dkim_log.getvalue(),))
if not dkim_result:
return None
# in addition to the dkim signature being valid, we have to check that it
# was actually signed by the user's domain.
if len(signing_details) != 1:
log.info(
'expected exactly one DKIM details record: %r'
% (signing_details,))
return None
signing_domain = signing_details[0]['d']
if not _isDkimDomainTrusted(signing_domain):
log.info("valid DKIM signature from untrusted domain %s"
% (signing_domain,))
return None
for origin in ['From', 'Sender']:
if signed_message[origin] is None:
continue
name, addr = parseaddr(signed_message[origin])
try:
origin_domain = addr.split('@')[1]
except IndexError:
log.warning(
"couldn't extract domain from address %r",
signed_message[origin])
if signing_domain == origin_domain:
log.info(
"DKIM signing domain %s matches %s address %r",
signing_domain, origin, addr)
return addr
else:
log.info("DKIM signing domain %s doesn't match message origin; "
"disregarding signature"
% (signing_domain))
return None
def test_stringio_bytes(self):
"""Verify that a Lark can be created from file-like objects other than Python's standard 'file' object"""
_Lark(cStringIO(b'start: a+ b a* "b" a*\n b: "b"\n a: "a" '))
def render_template(self, req, filename, data, content_type=None,
fragment=False):
"""Render the `filename` using the `data` for the context.
The `content_type` argument is used to choose the kind of template
used (TextTemplate if `'text/plain'`, MarkupTemplate otherwise), and
tweak the rendering process (use of XHTML Strict doctype if
`'text/html'` is given).
"""
if content_type is None:
content_type = 'text/html'
method = {'text/html': 'xhtml',
'text/plain': 'text'}.get(content_type, 'xml')
if method == "xhtml":
# Retrieve post-redirect messages saved in session
for type_ in ['warnings', 'notices']:
try:
for i in itertools.count():
message = req.session.pop('chrome.%s.%d' % (type_, i))
req.chrome[type_].append(Markup(message))
except KeyError:
pass
template = self.load_template(filename, method=method)
data = self.populate_data(req, data)
stream = template.generate(**data)
# Filter through ITemplateStreamFilter plugins
if self.stream_filters:
stream |= self._filter_stream(req, method, filename, stream, data)
if fragment:
return stream
if method == 'text':
buffer = cStringIO()
stream.render('text', out=buffer)
return buffer.getvalue()
doctype = {'text/html': DocType.XHTML_STRICT}.get(content_type)
if doctype:
if req.form_token:
stream |= self._add_form_token(req.form_token)
if not int(req.session.get('accesskeys', 0)):
stream |= self._strip_accesskeys
links = req.chrome.get('links')
scripts = req.chrome.get('scripts')
req.chrome['links'] = {}
req.chrome['scripts'] = []
data.setdefault('chrome', {}).update({
'late_links': req.chrome['links'],
'late_scripts': req.chrome['scripts'],
})
try:
buffer = cStringIO()
stream.render(method, doctype=doctype, out=buffer)
return buffer.getvalue().translate(_translate_nop,
_invalid_control_chars)
except Exception, e:
# restore what may be needed by the error template
req.chrome['links'] = links
req.chrome['scripts'] = scripts
# give some hints when hitting a Genshi unicode error
if isinstance(e, UnicodeError):
pos = self._stream_location(stream)
if pos:
location = "'%s', line %s, char %s" % pos
else:
location = _("(unknown template location)")
raise TracError(_("Genshi %(error)s error while rendering "
"template %(location)s",
error=e.__class__.__name__,
location=location))
raise
def string_io(data=None): # cStringIO can't handle unicode
try:
return cStringIO(bytes(data))
except UnicodeEncodeError:
return pyStringIO(data)
def mypng(graph): graph = worst_place_graph(df) f = cStringIO.cStringIO() graph.savefig(f, format='png') data = f.seek(0) return send_file(data, mimetype='image/png')
def test_python_cstringio():
"""Python cStringIO buffer"""
buffer = cStringIO()
_buffer_fn(buffer.write, TABLE_DATA)
return buffer.getvalue()
