def getHeaderAndContentsAsDictionaries(fd=sys.stdin, delimiter='\t'): contents = fd.read() contentsFileObj = cStringIO.cStringIO(contents) dr = csv.DictReader(contentsFileObj, delimiter=delimiter) rows = [row for row in dr] fd2 = cStringIO.cStringIO(contents) headerFields = fd2.readline().split(delimiter) fd2.close() return headerFields, rows
def test_defaults_os_x(self): sys.platform = 'darwin' reload(debops.config) cfgparser = configparser.SafeConfigParser() cfgparser.readfp(cStringIO(debops.config.DEFAULTS)) self.assertEqual(cfgparser.get('paths', 'data-home'), '~/Library/Application Support/debops')
def run(self, tasks, name): if self.status != RUNNING and self.todo: self.status = RUNNING else: return for parent in self.parents: if tasks[parent].getTodo(): self.status = WAITING return elif tasks[parent].getStatus() == FAILURE \ or tasks[parent].getStatus() == ABORT \ or not self.validate(tasks): self.status = ABORT self.todo = False if tasks[parent].getStatus() == FAILURE \ or tasks[parent].getStatus() == ABORT: self.abort_reason = "Parent error : "+parent else: self.abort_reason = "Bad return code from parent" return if self.status == RUNNING: child = {} try: print "START", name, time.strftime("%c") (child["in"], child["out"], child["err"]) = os.popen3(self.command) except Exception, (errno, strerror): child["err"] = cStringIO.cStringIO("-"+str(errno)+" - "+strerror+"-") self.status = FAILURE self.todo = False return self.returncode = child["err"].read() self.status = COMPLETION self.todo = False print "END", name, time.strftime("%c")
def test_read_numeric(): # make reader-like thing str_io = cStringIO() r = _make_readerlike(str_io) # check simplest of tags for base_dt, val, mdtype in ( ('u2', 30, mio5p.miUINT16), ('i4', 1, mio5p.miINT32), ('i2', -1, mio5p.miINT16)): for byte_code in ('<', '>'): r.byte_order = byte_code c_reader = m5u.VarReader5(r) yield assert_equal, c_reader.little_endian, byte_code == '<' yield assert_equal, c_reader.is_swapped, byte_code != boc.native_code for sde_f in (False, True): dt = np.dtype(base_dt).newbyteorder(byte_code) a = _make_tag(dt, val, mdtype, sde_f) a_str = a.tostring() _write_stream(str_io, a_str) el = c_reader.read_numeric() yield assert_equal, el, val # two sequential reads _write_stream(str_io, a_str, a_str) el = c_reader.read_numeric() yield assert_equal, el, val el = c_reader.read_numeric() yield assert_equal, el, val
def test_read_stream(): tag = _make_tag('i4', 1, mio5p.miINT32, sde=True) tag_str = tag.tostring() str_io = cStringIO(tag_str) st = streams.make_stream(str_io) s = streams._read_into(st, tag.itemsize) yield assert_equal, s, tag.tostring()
def test_zero_byte_string(): # Tests hack to allow chars of non-zero length, but 0 bytes # make reader-like thing str_io = cStringIO() r = _make_readerlike(str_io, '<') c_reader = m5u.VarReader5(r) tag_dt = np.dtype([('mdtype', 'u4'), ('byte_count', 'u4')]) tag = np.zeros((1,), dtype=tag_dt) tag['mdtype'] = mio5p.miINT8 tag['byte_count'] = 1 hdr = m5u.VarHeader5() # Try when string is 1 length hdr.set_dims([1,]) _write_stream(str_io, tag.tostring() + asbytes(' ')) str_io.seek(0) val = c_reader.read_char(hdr) assert_equal(val, u' ') # Now when string has 0 bytes 1 length tag['byte_count'] = 0 _write_stream(str_io, tag.tostring()) str_io.seek(0) val = c_reader.read_char(hdr) assert_equal(val, u' ') # Now when string has 0 bytes 4 length str_io.seek(0) hdr.set_dims([4,]) val = c_reader.read_char(hdr) assert_array_equal(val, [u' '] * 4)
def test_defaults_linux(self): sys.platform = 'linux2' reload(debops.config) cfgparser = configparser.SafeConfigParser() cfgparser.readfp(cStringIO(debops.config.DEFAULTS)) self.assertEqual(cfgparser.get('paths', 'data-home'), '$XDG_DATA_HOME/debops')
def make_xpi(files): file_obj = cStringIO() zip_file = zipfile.ZipFile(file_obj, 'w') for path, data in files.items(): zip_file.writestr(path, data) zip_file.close() file_obj.seek(0) return file_obj
def test_2StringIO(self): """ Python 2's L{StringIO} and L{cStringIO} modules are both binary I/O. """ from cStringIO import StringIO as cStringIO from StringIO import StringIO self.assertEqual(ioType(StringIO()), bytes) self.assertEqual(ioType(cStringIO()), bytes)
def test_2StringIO(self): """ Python 2's L{StringIO} and L{cStringIO} modules are both binary I/O. """ from cStringIO import StringIO as cStringIO from StringIO import StringIO self.assertEquals(ioType(StringIO()), bytes) self.assertEquals(ioType(cStringIO()), bytes)
def make_xpi(files): f = cStringIO() z = ZipFile(f, 'w') for path, data in files.items(): z.writestr(path, data) z.close() f.seek(0) return f
def string_io(data=None): # cStringIO can't handle unicode ''' Pass data through to stringIO module and return result ''' try: return cStringIO(bytes(data)) except UnicodeEncodeError: return pyStringIO(data)
def test_defaults_os_x(self): sys.platform = 'darwin' reload(debops.config) cfgparser = configparser.SafeConfigParser() cfgparser.readfp(cStringIO(debops.config.DEFAULTS)) self.assertEqual( cfgparser.get('paths', 'data-home'), os.path.expanduser('~/Library/Application Support/debops'))
def test_defaults_windows_without_APPDATA(self): sys.platform = 'win32' unsetenv('APPDATA') reload(debops.config) cfgparser = configparser.SafeConfigParser() cfgparser.readfp(cStringIO(debops.config.DEFAULTS)) self.assertEqual(cfgparser.get('paths', 'data-home'), '~\\Application Data/debops')
def test_defaults_windows_with_APPDATA(self): sys.platform = 'win32' setenv('APPDATA', 'H:\\my\\own\\data') reload(debops.config) cfgparser = configparser.SafeConfigParser() cfgparser.readfp(cStringIO(debops.config.DEFAULTS)) self.assertEqual(cfgparser.get('paths', 'data-home'), 'H:\\my\\own\\data/debops')
def process_response(self, request, response): if settings.DEBUG and 'prof' in request.GET: self.profiler.create_stats() io = cStringIO() stats = pstats.Stats(self.profiler, stream=io) stats.strip_dirs().sort_stats(request.GET.get('sort', 'time')) stats.print_stats(int(request.GET.get('count', 10000))) response.content = '<pre>%s</pre>' % io.getvalue() return response
def setup_test(): global streams val = b'a\x00string' fs = TemporaryFile() fs.write(val) fs.seek(0) gs = BytesIO(val) cs = cStringIO(val) streams = [fs, gs, cs]
def __init__(self, array_reader, byte_count): super(Mat5ZArrayReader, self).__init__( cStringIO(zlib.decompress( array_reader.mat_stream.read(byte_count))), array_reader.dtypes, array_reader.processor_func, array_reader.codecs, array_reader.class_dtypes, array_reader.struct_as_record)
def prepare_transform(self, data, arguments=None): """ The method takes some data in one of the input formats and returns a TransformResult with data in the output format. """ if arguments is not None: infile_data_suffix = arguments.get('infile_data_suffix', False) del arguments['infile_data_suffix'] result = TransformResult(None) tmpfilepath = None try: tmpdirpath = tempfile.mkdtemp() tmpfilepath = self.initialize_tmpfile(data, directory=tmpdirpath) if infile_data_suffix: primaryname = os.path.basename(tmpfilepath) + infile_data_suffix commandline = 'cd "%s" && %s %s' % ( tmpdirpath, self.command, self.args) arguments['infile'] = tmpfilepath commandline = commandline % arguments if os.name=='posix': # TODO: tbenita suggests to remove 1>/dev/null as some commands # return result in flow. Maybe turn this into another subobject # commandline = commandline + ' 2>error_log' commandline = commandline + ' 2>error_log 1>/dev/null' os.system(commandline) for tmpfile in os.listdir(tmpdirpath): tmp = os.path.join(tmpdirpath, tmpfile) # Exclude the original file and the error_log from the result if tmp == tmpfilepath: continue fd = file(tmp, 'rb') # Should we use the infile as the primary output? if infile_data_suffix and primaryname == tmpfile: result.data = StringIO() result.data.writelines(fd) result.data.seek(0) elif tmp.endswith('error_log'): result.errors = fd.read() else: sub = cStringIO() sub.writelines(fd) sub.seek(0) result.subobjects[tmpfile] = sub fd.close() os.unlink(tmp) finally: if tmpfilepath is not None and os.path.isdir(tmpdirpath): shutil.rmtree(tmpdirpath) return result
def _update_caps(self, caps): # Rewrite the moov in the streamheaders buffer raw_headers = cStringIO() iso.write_atoms([self._moov], raw_headers) self._streamheaders = gst.Buffer(raw_headers.getvalue()) self._streamheaders.flag_set(gst.BUFFER_FLAG_IN_CAPS) s = caps[0].copy() s.set_value('streamheader', (self._streamheaders, )) self._outputcaps = gst.Caps(s) self._streamheaders.set_caps(self._outputcaps)
def text2wave(text): process = subprocess.Popen(_text2wave, stdin=subprocess.PIPE, stdout=subprocess.PIPE) try: process.stdin.write(text + os.linesep) except TypeError: process.stdin.write(bytes(text + os.linesep, "utf-8")) (output, err) = process.communicate() exitCode = process.wait() wavf = cStringIO(output) return wave.open(wavf, "rb")
def setup(): val = b'a\x00string' global fs, gs, cs, fname fd, fname = mkstemp() fs = os.fdopen(fd, 'wb') fs.write(val) fs.close() fs = open(fname, 'rb') gs = BytesIO(val) cs = cStringIO(val)
def setup_test_file(): val = b'a\x00string' fd, fname = mkstemp() with os.fdopen(fd, 'wb') as fs: fs.write(val) with open(fname, 'rb') as fs: gs = BytesIO(val) cs = cStringIO(val) yield fs, gs, cs os.unlink(fname)
def test_read_numeric_writeable(): # make reader-like thing str_io = cStringIO() r = _make_readerlike(str_io, '<') c_reader = m5u.VarReader5(r) dt = np.dtype('<u2') a = _make_tag(dt, 30, mio5p.miUINT16, 0) a_str = a.tostring() _write_stream(str_io, a_str) el = c_reader.read_numeric() yield assert_true, el.flags.writeable
def test_write_to(self): b = self.nested_buffer() fp = cStringIO() b.write_to(fp, up_to = b.world, close=True) self.assertEqual("Hallo", fp.getvalue()) def r(buff): buff.write(" ") # The first buffer may not be written to any more. self.assertRaises(IllegalFunctionCall, r, b.hello)
def test_read_numeric_writeable(): # make reader-like thing str_io = cStringIO() r = _make_readerlike() r.mat_stream = str_io r.byte_order = '<' r.dtypes = miob.convert_dtypes(mio5.mdtypes_template, '<') c_reader = m5u.VarReader5(r) dt = np.dtype('<u2') a = _make_tag(dt, 30, mio5.miUINT16, 0) a_str = a.tostring() _write_stream(str_io, a_str) el = c_reader.read_numeric() yield assert_true, el.flags.writeable
def save_mail_to_librarian(raw_mail, restricted=False): """Save the message to the librarian. It can be referenced from errors, and also accessed by code that needs to get back the exact original form. """ # File the raw_mail in the Librarian. We generate a filename to avoid # people guessing the URL. We don't want URLs to private bug messages to # be guessable for example. file_name = str(uuid1()) + '.txt' file_alias = getUtility(ILibraryFileAliasSet).create( file_name, len(raw_mail), cStringIO(raw_mail), 'message/rfc822', restricted=restricted) return file_alias
def run(self, tasks, name): if self.status != RUNNING and self.todo: self.status = RUNNING else: return for parent in self.parents: if tasks[parent].getTodo(): self.status = WAITING return elif tasks[parent].getStatus() == FAILURE \ or tasks[parent].getStatus() == ABORT \ or not self.validate(tasks): self.status = ABORT self.todo = False if tasks[parent].getStatus() == FAILURE \ or tasks[parent].getStatus() == ABORT: self.abort_reason = "Parent error : " + parent else: self.abort_reason = "Bad return code from parent" return if self.status == RUNNING: child = {} try: print "START", name, time.strftime("%c") (child["in"], child["out"], child["err"]) = os.popen3(self.command) except Exception, (errno, strerror): child["err"] = cStringIO.cStringIO("-" + str(errno) + " - " + strerror + "-") self.status = FAILURE self.todo = False return self.returncode = child["err"].read() self.status = COMPLETION self.todo = False print "END", name, time.strftime("%c")
def _parse_headers(self, buf, tid): # Parse buffer and get the moov atom f = cStringIO(buf.data) al = list(atoms.read_atoms(f)) ad = atoms.atoms_dict(al) try: moov = iso.select_atoms(ad, ('moov', 1, 1))[0] except Exception: self.error("Could not parse moov") m = messages.Error(T_(N_( "First buffer cannot be parsed as a moov. "\ "The required mp4seek version is 1.0-6"))) self.addMessage(m) return False # Modify the track id and update the streamheaders buffer trak = moov.trak[0] trak.tkhd.id = tid if self._moov is None: self._moov = moov else: self._moov.trak.append(trak) self._update_caps(buf.caps) return True
def StringIO(s=None): # cStringIO can't handle unicode try: return cStringIO(bytes(s)) except UnicodeEncodeError: return pyStringIO(s)
def _verifyDkimOrigin(signed_message): """Find a From or Sender address for which there's a DKIM signature. :returns: A string email address for the trusted sender, if there is one, otherwise None. :param signed_message: ISignedMessage """ log = logging.getLogger('mail-authenticate-dkim') log.setLevel(logging.DEBUG) if getFeatureFlag('mail.dkim_authentication.disabled'): log.info('dkim authentication feature disabled') return None # uncomment this for easier test debugging # log.addHandler(logging.FileHandler('/tmp/dkim.log')) dkim_log = cStringIO() log.info( 'Attempting DKIM authentication of message id=%r from=%r sender=%r' % (signed_message['Message-ID'], signed_message['From'], signed_message['Sender'])) signing_details = [] dkim_result = False try: dkim_result = dkim.verify(signed_message.parsed_string, dkim_log, details=signing_details) except dkim.DKIMException as e: log.warning('DKIM error: %r' % (e, )) except dns.resolver.NXDOMAIN as e: # This can easily happen just through bad input data, ie claiming to # be signed by a domain with no visible key of that name. It's not an # operational error. log.info('DNS exception: %r' % (e, )) except dns.exception.DNSException as e: # many of them have lame messages, thus %r log.warning('DNS exception: %r' % (e, )) except Exception as e: # DKIM leaks some errors when it gets bad input, as in bug 881237. We # don't generally want them to cause the mail to be dropped entirely # though. It probably is reasonable to treat them as potential # operational errors, at least until they're handled properly, by # making pydkim itself more defensive. log.warning( 'unexpected error in DKIM verification, treating as unsigned: %r' % (e, )) log.info('DKIM verification result: trusted=%s' % (dkim_result, )) log.debug('DKIM debug log: %s' % (dkim_log.getvalue(), )) if not dkim_result: return None # in addition to the dkim signature being valid, we have to check that it # was actually signed by the user's domain. if len(signing_details) != 1: log.info('expected exactly one DKIM details record: %r' % (signing_details, )) return None signing_domain = signing_details[0]['d'] if not _isDkimDomainTrusted(signing_domain): log.info("valid DKIM signature from untrusted domain %s" % (signing_domain, )) return None for origin in ['From', 'Sender']: if signed_message[origin] is None: continue name, addr = parseaddr(signed_message[origin]) try: origin_domain = addr.split('@')[1] except IndexError: log.warning("couldn't extract domain from address %r", signed_message[origin]) if signing_domain == origin_domain: log.info("DKIM signing domain %s matches %s address %r", signing_domain, origin, addr) return addr else: log.info("DKIM signing domain %s doesn't match message origin; " "disregarding signature" % (signing_domain)) return None
def getImage(): stream = cStringIO.cStringIO(data) return wx.ImageFromStream( stream )
def fromEmail(self, email_message, owner=None, filealias=None, parsed_message=None, create_missing_persons=False, fallback_parent=None, date_created=None, restricted=False): """See IMessageSet.fromEmail.""" # It does not make sense to handle Unicode strings, as email # messages may contain chunks encoded in differing character sets. # Passing Unicode in here indicates a bug. if not zisinstance(email_message, str): raise TypeError('email_message must be a normal string. Got: %r' % email_message) # Parse the raw message into an email.message.Message instance, # if we haven't been given one already. if parsed_message is None: parsed_message = email.message_from_string(email_message) # We could easily generate a default, but a missing message-id # almost certainly means a developer is using this method when # they shouldn't (by creating emails by hand and passing them here), # which is broken because they will almost certainly have Unicode # errors. rfc822msgid = parsed_message.get('message-id') if not rfc822msgid: raise InvalidEmailMessage('Missing Message-Id') # Over-long messages are checked for at the handle_on_message level. # If it's a restricted mail (IE: for a private bug), or it hasn't been # uploaded, do so now. from lp.services.mail.helpers import save_mail_to_librarian if restricted or filealias is None: raw_email_message = save_mail_to_librarian(email_message, restricted=restricted) else: raw_email_message = filealias # Find the message subject subject = self._decode_header(parsed_message.get('subject', '')) subject = subject.strip() if owner is None: # Try and determine the owner. We raise a NotFoundError # if the sender does not exist, unless we were asked to # create_missing_persons. person_set = getUtility(IPersonSet) from_hdr = self._decode_header(parsed_message.get('from', '')).strip() replyto_hdr = self._decode_header( parsed_message.get('reply-to', '')).strip() from_addrs = [from_hdr, replyto_hdr] from_addrs = [parseaddr(addr) for addr in from_addrs if addr] if len(from_addrs) == 0: raise InvalidEmailMessage('No From: or Reply-To: header') for from_addr in from_addrs: owner = person_set.getByEmail(from_addr[1].lower().strip()) if owner is not None: break if owner is None: if not create_missing_persons: raise UnknownSender(from_addrs[0][1]) # autocreate a person sendername = ensure_unicode(from_addrs[0][0].strip()) senderemail = from_addrs[0][1].lower().strip() # XXX: Guilherme Salgado 2006-08-31 bug=62344: # It's hard to define what rationale to use here, and to # make things worst, it's almost impossible to provide a # meaningful comment having only the email message. owner = person_set.ensurePerson( senderemail, sendername, PersonCreationRationale.FROMEMAILMESSAGE) if owner is None: raise UnknownSender(senderemail) # Get the parent of the message, if available in the db. We'll # go through all the message's parents until we find one that's # in the db. parent = None for parent_msgid in reversed(get_parent_msgids(parsed_message)): try: # we assume it's the first matching message parent = self.get(parent_msgid)[0] break except NotFoundError: pass else: parent = fallback_parent # Figure out the date of the message. if date_created is not None: datecreated = date_created else: datecreated = utcdatetime_from_field(parsed_message['date']) # Make sure we don't create an email with a datecreated in the # distant past or future. now = datetime.now(pytz.timezone('UTC')) thedistantpast = datetime(1990, 1, 1, tzinfo=pytz.timezone('UTC')) if datecreated < thedistantpast or datecreated > now: datecreated = UTC_NOW message = Message(subject=subject, owner=owner, rfc822msgid=rfc822msgid, parent=parent, raw=raw_email_message, datecreated=datecreated) sequence = 1 # Don't store the preamble or epilogue -- they are only there # to give hints to non-MIME aware clients # # Determine the encoding to use for non-multipart messages, and the # preamble and epilogue of multipart messages. We default to # iso-8859-1 as it seems fairly harmless to cope with old, broken # mail clients (The RFCs state US-ASCII as the default character # set). # default_charset = (parsed_message.get_content_charset() or # 'iso-8859-1') # # XXX: kiko 2005-09-23: Is default_charset only useful here? # # if getattr(parsed_message, 'preamble', None): # # We strip a leading and trailing newline - the email parser # # seems to arbitrarily add them :-/ # preamble = parsed_message.preamble.decode( # default_charset, 'replace') # if preamble.strip(): # if preamble[0] == '\n': # preamble = preamble[1:] # if preamble[-1] == '\n': # preamble = preamble[:-1] # MessageChunk( # message=message, sequence=sequence, content=preamble # ) # sequence += 1 for part in parsed_message.walk(): mime_type = part.get_content_type() # Skip the multipart section that walk gives us. This part # is the entire message. if part.is_multipart(): continue # Decode the content of this part. content = part.get_payload(decode=True) # Store the part as a MessageChunk # # We want only the content type text/plain as "main content". # Exceptions to this rule: # - if the content disposition header explicitly says that # this part is an attachment, text/plain content is stored # as a blob, # - if the content-disposition header provides a filename, # text/plain content is stored as a blob. content_disposition = part.get('Content-disposition', '').lower() no_attachment = not content_disposition.startswith('attachment') if (mime_type == 'text/plain' and no_attachment and part.get_filename() is None): # Get the charset for the message part. If one isn't # specified, default to latin-1 to prevent # UnicodeDecodeErrors. charset = part.get_content_charset() if charset is None or str(charset).lower() == 'x-unknown': charset = 'latin-1' content = self.decode(content, charset) if content.strip(): MessageChunk(message=message, sequence=sequence, content=content) sequence += 1 else: filename = part.get_filename() or 'unnamed' # Strip off any path information. filename = os.path.basename(filename) # Note we use the Content-Type header instead of # part.get_content_type() here to ensure we keep # parameters as sent. If Content-Type is None we default # to application/octet-stream. if part['content-type'] is None: content_type = 'application/octet-stream' else: content_type = part['content-type'] if len(content) > 0: blob = getUtility(ILibraryFileAliasSet).create( name=filename, size=len(content), file=cStringIO(content), contentType=content_type, restricted=restricted) MessageChunk(message=message, sequence=sequence, blob=blob) sequence += 1 # Don't store the epilogue # if getattr(parsed_message, 'epilogue', None): # epilogue = parsed_message.epilogue.decode( # default_charset, 'replace') # if epilogue.strip(): # if epilogue[0] == '\n': # epilogue = epilogue[1:] # if epilogue[-1] == '\n': # epilogue = epilogue[:-1] # MessageChunk( # message=message, sequence=sequence, content=epilogue # ) # XXX 2008-05-27 jamesh: # Ensure that BugMessages get flushed in same order as they # are created. Store.of(message).flush() return message
def fromEmail(self, email_message, owner=None, filealias=None, parsed_message=None, create_missing_persons=False, fallback_parent=None, date_created=None, restricted=False): """See IMessageSet.fromEmail.""" # It does not make sense to handle Unicode strings, as email # messages may contain chunks encoded in differing character sets. # Passing Unicode in here indicates a bug. if not zisinstance(email_message, str): raise TypeError( 'email_message must be a normal string. Got: %r' % email_message) # Parse the raw message into an email.Message.Message instance, # if we haven't been given one already. if parsed_message is None: parsed_message = email.message_from_string(email_message) # We could easily generate a default, but a missing message-id # almost certainly means a developer is using this method when # they shouldn't (by creating emails by hand and passing them here), # which is broken because they will almost certainly have Unicode # errors. rfc822msgid = parsed_message.get('message-id') if not rfc822msgid: raise InvalidEmailMessage('Missing Message-Id') # Over-long messages are checked for at the handle_on_message level. # If it's a restricted mail (IE: for a private bug), or it hasn't been # uploaded, do so now. from lp.services.mail.helpers import save_mail_to_librarian if restricted or filealias is None: raw_email_message = save_mail_to_librarian( email_message, restricted=restricted) else: raw_email_message = filealias # Find the message subject subject = self._decode_header(parsed_message.get('subject', '')) subject = subject.strip() if owner is None: # Try and determine the owner. We raise a NotFoundError # if the sender does not exist, unless we were asked to # create_missing_persons. person_set = getUtility(IPersonSet) from_hdr = self._decode_header( parsed_message.get('from', '')).strip() replyto_hdr = self._decode_header( parsed_message.get('reply-to', '')).strip() from_addrs = [from_hdr, replyto_hdr] from_addrs = [parseaddr(addr) for addr in from_addrs if addr] if len(from_addrs) == 0: raise InvalidEmailMessage('No From: or Reply-To: header') for from_addr in from_addrs: owner = person_set.getByEmail(from_addr[1].lower().strip()) if owner is not None: break if owner is None: if not create_missing_persons: raise UnknownSender(from_addrs[0][1]) # autocreate a person sendername = ensure_unicode(from_addrs[0][0].strip()) senderemail = from_addrs[0][1].lower().strip() # XXX: Guilherme Salgado 2006-08-31 bug=62344: # It's hard to define what rationale to use here, and to # make things worst, it's almost impossible to provide a # meaningful comment having only the email message. owner = person_set.ensurePerson( senderemail, sendername, PersonCreationRationale.FROMEMAILMESSAGE) if owner is None: raise UnknownSender(senderemail) # Get the parent of the message, if available in the db. We'll # go through all the message's parents until we find one that's # in the db. parent = None for parent_msgid in reversed(get_parent_msgids(parsed_message)): try: # we assume it's the first matching message parent = self.get(parent_msgid)[0] break except NotFoundError: pass else: parent = fallback_parent # Figure out the date of the message. if date_created is not None: datecreated = date_created else: datecreated = utcdatetime_from_field(parsed_message['date']) # Make sure we don't create an email with a datecreated in the # distant past or future. now = datetime.now(pytz.timezone('UTC')) thedistantpast = datetime(1990, 1, 1, tzinfo=pytz.timezone('UTC')) if datecreated < thedistantpast or datecreated > now: datecreated = UTC_NOW message = Message(subject=subject, owner=owner, rfc822msgid=rfc822msgid, parent=parent, raw=raw_email_message, datecreated=datecreated) sequence = 1 # Don't store the preamble or epilogue -- they are only there # to give hints to non-MIME aware clients # # Determine the encoding to use for non-multipart messages, and the # preamble and epilogue of multipart messages. We default to # iso-8859-1 as it seems fairly harmless to cope with old, broken # mail clients (The RFCs state US-ASCII as the default character # set). # default_charset = (parsed_message.get_content_charset() or # 'iso-8859-1') # # XXX: kiko 2005-09-23: Is default_charset only useful here? # # if getattr(parsed_message, 'preamble', None): # # We strip a leading and trailing newline - the email parser # # seems to arbitrarily add them :-/ # preamble = parsed_message.preamble.decode( # default_charset, 'replace') # if preamble.strip(): # if preamble[0] == '\n': # preamble = preamble[1:] # if preamble[-1] == '\n': # preamble = preamble[:-1] # MessageChunk( # message=message, sequence=sequence, content=preamble # ) # sequence += 1 for part in parsed_message.walk(): mime_type = part.get_content_type() # Skip the multipart section that walk gives us. This part # is the entire message. if part.is_multipart(): continue # Decode the content of this part. content = part.get_payload(decode=True) # Store the part as a MessageChunk # # We want only the content type text/plain as "main content". # Exceptions to this rule: # - if the content disposition header explicitly says that # this part is an attachment, text/plain content is stored # as a blob, # - if the content-disposition header provides a filename, # text/plain content is stored as a blob. content_disposition = part.get('Content-disposition', '').lower() no_attachment = not content_disposition.startswith('attachment') if (mime_type == 'text/plain' and no_attachment and part.get_filename() is None): # Get the charset for the message part. If one isn't # specified, default to latin-1 to prevent # UnicodeDecodeErrors. charset = part.get_content_charset() if charset is None or str(charset).lower() == 'x-unknown': charset = 'latin-1' content = self.decode(content, charset) if content.strip(): MessageChunk( message=message, sequence=sequence, content=content) sequence += 1 else: filename = part.get_filename() or 'unnamed' # Strip off any path information. filename = os.path.basename(filename) # Note we use the Content-Type header instead of # part.get_content_type() here to ensure we keep # parameters as sent. If Content-Type is None we default # to application/octet-stream. if part['content-type'] is None: content_type = 'application/octet-stream' else: content_type = part['content-type'] if len(content) > 0: blob = getUtility(ILibraryFileAliasSet).create( name=filename, size=len(content), file=cStringIO(content), contentType=content_type, restricted=restricted) MessageChunk(message=message, sequence=sequence, blob=blob) sequence += 1 # Don't store the epilogue # if getattr(parsed_message, 'epilogue', None): # epilogue = parsed_message.epilogue.decode( # default_charset, 'replace') # if epilogue.strip(): # if epilogue[0] == '\n': # epilogue = epilogue[1:] # if epilogue[-1] == '\n': # epilogue = epilogue[:-1] # MessageChunk( # message=message, sequence=sequence, content=epilogue # ) # XXX 2008-05-27 jamesh: # Ensure that BugMessages get flushed in same order as they # are created. Store.of(message).flush() return message
def convert(images, dpi=None, pagesize=(None, None), title=None, author=None, creator=None, producer=None, creationdate=None, moddate=None, subject=None, keywords=None, colorspace=None, nodate=False, verbose=False): pdf = pdfdoc(3, title, author, creator, producer, creationdate, moddate, subject, keywords, nodate) for imfilename in images: debug_out("Reading %s"%imfilename, verbose) try: rawdata = imfilename.read() except AttributeError: with open(imfilename, "rb") as im: rawdata = im.read() im = cStringIO(rawdata) try: imgdata = Image.open(im) except IOError as e: # test if it is a jpeg2000 image if rawdata[:12] != "\x00\x00\x00\x0C\x6A\x50\x20\x20\x0D\x0A\x87\x0A": error_out("cannot read input image (not jpeg2000)") error_out("PIL: %s"%e) exit(1) # image is jpeg2000 width, height, ics = parsejp2(rawdata) imgformat = "JPEG2000" if dpi: ndpi = dpi, dpi debug_out("input dpi (forced) = %d x %d"%ndpi, verbose) else: # TODO: read real dpi from input jpeg2000 image ndpi = (96, 96) debug_out("input dpi = %d x %d"%ndpi, verbose) if colorspace: color = colorspace debug_out("input colorspace (forced) = %s"%(ics)) else: color = ics debug_out("input colorspace = %s"%(ics), verbose) else: width, height = imgdata.size imgformat = imgdata.format if dpi: ndpi = dpi, dpi debug_out("input dpi (forced) = %d x %d"%ndpi, verbose) else: ndpi = imgdata.info.get("dpi", (96, 96)) # in python3, the returned dpi value for some tiff images will # not be an integer but a float. To make the behaviour of # img2pdf the same between python2 and python3, we convert that # float into an integer by rounding # search online for the 72.009 dpi problem for more info ndpi = (int(round(ndpi[0])),int(round(ndpi[1]))) debug_out("input dpi = %d x %d"%ndpi, verbose) if colorspace: color = colorspace debug_out("input colorspace (forced) = %s"%(color), verbose) else: color = imgdata.mode if color == "CMYK" and imgformat == "JPEG": # Adobe inverts CMYK JPEGs for some reason, and others # have followed suit as well. Some software assumes the # JPEG is inverted if the Adobe tag (APP14), while other # software assumes all CMYK JPEGs are inverted. I don't # have enough experience with these to know which is # better for images currently in the wild, so I'm going # with the first approach for now. if "adobe" in imgdata.info: color = "CMYK;I" debug_out("input colorspace = %s"%(color), verbose) debug_out("width x height = %d x %d"%(width,height), verbose) debug_out("imgformat = %s"%imgformat, verbose) # depending on the input format, determine whether to pass the raw # image or the zlib compressed color information if imgformat is "JPEG" or imgformat is "JPEG2000": if color == '1': error_out("jpeg can't be monochrome") exit(1) imgdata = rawdata else: # because we do not support /CCITTFaxDecode if color == '1': debug_out("Converting colorspace 1 to L", verbose) imgdata = imgdata.convert('L') color = 'L' elif color in ("RGB", "L", "CMYK", "CMYK;I"): debug_out("Colorspace is OK: %s"%color, verbose) else: debug_out("Converting colorspace %s to RGB"%color, verbose) imgdata = imgdata.convert('RGB') color = imgdata.mode img = imgdata.tobytes() # the python-pil version 2.3.0-1ubuntu3 in Ubuntu does not have the close() method try: imgdata.close() except AttributeError: pass imgdata = zlib.compress(img) im.close() # pdf units = 1/72 inch if not pagesize[0] and not pagesize[1]: pdf_x, pdf_y = 72.0*width/float(ndpi[0]), 72.0*height/float(ndpi[1]) elif not pagesize[1]: pdf_x, pdf_y = pagesize[0], pagesize[0]*height/float(width) elif not pagesize[0]: pdf_x, pdf_y = pagesize[1]*width/float(height), pagesize[1] else: pdf_x = pagesize[0] pdf_y = pagesize[1] pdf.addimage(color, width, height, imgformat, imgdata, pdf_x, pdf_y) return pdf.tostring()
def from_string(cls, s): """ Create a dsc_document from string s. """ return cls.from_file(cStringIO(s))
def test_render_output_stream_utf8(self): xml = XML('<li>Über uns</li>') strio = cStringIO() self.assertEqual(None, xml.render(out=strio)) self.assertEqual('<li>Über uns</li>', strio.getvalue())
def test_stringio_bytes(self): """Verify that a Grammar can be created from file-like objects other than Python's standard 'file' object""" Grammar(cStringIO(b"start: a+ b a+? 'b' a*; b: 'b'; a: 'a';"))
def _verifyDkimOrigin(signed_message): """Find a From or Sender address for which there's a DKIM signature. :returns: A string email address for the trusted sender, if there is one, otherwise None. :param signed_message: ISignedMessage """ log = logging.getLogger('mail-authenticate-dkim') log.setLevel(logging.DEBUG) if getFeatureFlag('mail.dkim_authentication.disabled'): log.info('dkim authentication feature disabled') return None # uncomment this for easier test debugging # log.addHandler(logging.FileHandler('/tmp/dkim.log')) dkim_log = cStringIO() log.info( 'Attempting DKIM authentication of message id=%r from=%r sender=%r' % (signed_message['Message-ID'], signed_message['From'], signed_message['Sender'])) signing_details = [] dkim_result = False try: dkim_result = dkim.verify( signed_message.parsed_string, dkim_log, details=signing_details) except dkim.DKIMException as e: log.warning('DKIM error: %r' % (e,)) except dns.resolver.NXDOMAIN as e: # This can easily happen just through bad input data, ie claiming to # be signed by a domain with no visible key of that name. It's not an # operational error. log.info('DNS exception: %r' % (e,)) except dns.exception.DNSException as e: # many of them have lame messages, thus %r log.warning('DNS exception: %r' % (e,)) except Exception as e: # DKIM leaks some errors when it gets bad input, as in bug 881237. We # don't generally want them to cause the mail to be dropped entirely # though. It probably is reasonable to treat them as potential # operational errors, at least until they're handled properly, by # making pydkim itself more defensive. log.warning( 'unexpected error in DKIM verification, treating as unsigned: %r' % (e,)) log.info('DKIM verification result: trusted=%s' % (dkim_result,)) log.debug('DKIM debug log: %s' % (dkim_log.getvalue(),)) if not dkim_result: return None # in addition to the dkim signature being valid, we have to check that it # was actually signed by the user's domain. if len(signing_details) != 1: log.info( 'expected exactly one DKIM details record: %r' % (signing_details,)) return None signing_domain = signing_details[0]['d'] if not _isDkimDomainTrusted(signing_domain): log.info("valid DKIM signature from untrusted domain %s" % (signing_domain,)) return None for origin in ['From', 'Sender']: if signed_message[origin] is None: continue name, addr = parseaddr(signed_message[origin]) try: origin_domain = addr.split('@')[1] except IndexError: log.warning( "couldn't extract domain from address %r", signed_message[origin]) if signing_domain == origin_domain: log.info( "DKIM signing domain %s matches %s address %r", signing_domain, origin, addr) return addr else: log.info("DKIM signing domain %s doesn't match message origin; " "disregarding signature" % (signing_domain)) return None
def test_stringio_bytes(self): """Verify that a Lark can be created from file-like objects other than Python's standard 'file' object""" _Lark(cStringIO(b'start: a+ b a* "b" a*\n b: "b"\n a: "a" '))
def render_template(self, req, filename, data, content_type=None, fragment=False): """Render the `filename` using the `data` for the context. The `content_type` argument is used to choose the kind of template used (TextTemplate if `'text/plain'`, MarkupTemplate otherwise), and tweak the rendering process (use of XHTML Strict doctype if `'text/html'` is given). """ if content_type is None: content_type = 'text/html' method = {'text/html': 'xhtml', 'text/plain': 'text'}.get(content_type, 'xml') if method == "xhtml": # Retrieve post-redirect messages saved in session for type_ in ['warnings', 'notices']: try: for i in itertools.count(): message = req.session.pop('chrome.%s.%d' % (type_, i)) req.chrome[type_].append(Markup(message)) except KeyError: pass template = self.load_template(filename, method=method) data = self.populate_data(req, data) stream = template.generate(**data) # Filter through ITemplateStreamFilter plugins if self.stream_filters: stream |= self._filter_stream(req, method, filename, stream, data) if fragment: return stream if method == 'text': buffer = cStringIO() stream.render('text', out=buffer) return buffer.getvalue() doctype = {'text/html': DocType.XHTML_STRICT}.get(content_type) if doctype: if req.form_token: stream |= self._add_form_token(req.form_token) if not int(req.session.get('accesskeys', 0)): stream |= self._strip_accesskeys links = req.chrome.get('links') scripts = req.chrome.get('scripts') req.chrome['links'] = {} req.chrome['scripts'] = [] data.setdefault('chrome', {}).update({ 'late_links': req.chrome['links'], 'late_scripts': req.chrome['scripts'], }) try: buffer = cStringIO() stream.render(method, doctype=doctype, out=buffer) return buffer.getvalue().translate(_translate_nop, _invalid_control_chars) except Exception, e: # restore what may be needed by the error template req.chrome['links'] = links req.chrome['scripts'] = scripts # give some hints when hitting a Genshi unicode error if isinstance(e, UnicodeError): pos = self._stream_location(stream) if pos: location = "'%s', line %s, char %s" % pos else: location = _("(unknown template location)") raise TracError(_("Genshi %(error)s error while rendering " "template %(location)s", error=e.__class__.__name__, location=location)) raise
def string_io(data=None): # cStringIO can't handle unicode try: return cStringIO(bytes(data)) except UnicodeEncodeError: return pyStringIO(data)
def mypng(graph): graph = worst_place_graph(df) f = cStringIO.cStringIO() graph.savefig(f, format='png') data = f.seek(0) return send_file(data, mimetype='image/png')
def test_python_cstringio(): """Python cStringIO buffer""" buffer = cStringIO() _buffer_fn(buffer.write, TABLE_DATA) return buffer.getvalue()