Example #1
0
def upgrade():

    from inbox.server.config import load_config
    load_config()

    from inbox.server.models import session_scope, Session
    from inbox.server.models.ignition import engine

    from inbox.server.models.tables.base import (Part, Namespace, Message,
                                                 Thread)
    from inbox.sqlalchemy.util import JSON

    print 'Creating table for parts...'
    op.create_table(
        'part', sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('message_id', sa.Integer(), nullable=True),
        sa.Column('walk_index', sa.Integer(), nullable=True),
        sa.Column('content_disposition',
                  sa.Enum('inline', 'attachment'),
                  nullable=True),
        sa.Column('content_id', sa.String(length=255), nullable=True),
        sa.Column('misc_keyval', JSON(), nullable=True),
        sa.Column('is_inboxapp_attachment',
                  sa.Boolean(),
                  server_default=sa.sql.expression.false(),
                  nullable=True),
        sa.ForeignKeyConstraint(['id'], ['block.id'], ondelete='CASCADE'),
        sa.ForeignKeyConstraint(['message_id'], ['message.id'],
                                ondelete='CASCADE'),
        sa.PrimaryKeyConstraint('id'),
        sa.UniqueConstraint('message_id', 'walk_index'))

    print 'Reflecting old block table schema'
    Base = declarative_base()
    Base.metadata.reflect(engine)

    class Block_(Base):  # old schema, reflected from database table
        __table__ = Base.metadata.tables['block']

    print 'Adding namespace_id column to blocks ',
    op.add_column(u'block',
                  sa.Column('namespace_id', sa.Integer(), nullable=False))

    print 'Migrating from blocks to parts'
    new_parts = []
    with session_scope() as db_session:
        for block in db_session.query(Block_).yield_per(chunk_size):

            # Move relevant fields
            p = Part()
            p.size = block.size
            p.data_sha256 = block.data_sha256
            p.message_id = block.message_id
            p.walk_index = block.walk_index
            p.content_disposition = block.content_disposition
            p.content_id = block.content_id
            p.misc_keyval = block.misc_keyval
            p.is_inboxapp_attachment

            old_namespace = db_session.query(Namespace) \
                .join(Message.thread, Thread.namespace) \
                .filter(Message.id == block.message_id).one()
            p.namespace_id = old_namespace.id

            # Commit after column modifications
            new_parts.append(p)

        print 'Deleting old blocks (now parts)... ',
        db_session.query(Block_).delete()
        db_session.commit()
        print 'Done!'

    print 'Removing `message_id` constraint from block'
    op.drop_constraint('block_ibfk_1', 'block', type_='foreignkey')

    print 'Creating foreign key for block -> namespace on block'
    op.create_foreign_key('block_ibfk_1',
                          'block',
                          'namespace', ['namespace_id'], ['id'],
                          ondelete='CASCADE')

    print 'Dropping old block columns which are now in part'
    op.drop_column(u'block', u'walk_index')
    op.drop_column(u'block', u'content_disposition')
    op.drop_column(u'block', u'misc_keyval')
    op.drop_column(u'block', u'content_id')
    op.drop_column(u'block', u'is_inboxapp_attachment')
    op.drop_constraint(u'message_id', 'block', type_='unique')
    op.drop_column(u'block', u'message_id')

    # Note: here we use the regular database session, since the transaction
    # log requires the `namespace` property on objects. We've set the
    # `namespace_id` foreign key, but need to commit the object before the
    # SQLalchemy reference is valid
    no_tx_session = Session(autoflush=True, autocommit=False)
    no_tx_session.add_all(new_parts)
    no_tx_session.commit()

    print 'Done migration blocks to parts!'
Example #2
0
def create_message(db_session, log, account, mid, folder_name, received_date,
                   flags, body_string, created):
    """ Parses message data and writes out db metadata and MIME blocks.

    Returns the new Message, which links to the new Block objects through
    relationships. All new objects are uncommitted.

    Threads are not computed here; you gotta do that separately.

    Parameters
    ----------
    mid : int
        The account backend-specific message identifier; it's only used for
        logging errors.

    raw_message : str
        The full message including headers (encoded).
    """
    # trickle-down bugs
    assert account is not None and account.namespace is not None
    assert not isinstance(body_string, unicode)

    try:
        parsed = mime.from_string(body_string)

        mime_version = parsed.headers.get('Mime-Version')
        # NOTE: sometimes MIME-Version is set to "1.0 (1.0)", hence the
        # .startswith
        if mime_version is not None and not mime_version.startswith('1.0'):
            log.error('Unexpected MIME-Version: {0}'.format(mime_version))

        new_msg = SpoolMessage() if created else Message()
        new_msg.data_sha256 = sha256(body_string).hexdigest()

        # clean_subject strips re:, fwd: etc.
        new_msg.subject = parsed.clean_subject
        new_msg.from_addr = parse_email_address_list(
            parsed.headers.get('From'))
        new_msg.sender_addr = parse_email_address_list(
            parsed.headers.get('Sender'))
        new_msg.reply_to = parse_email_address_list(
            parsed.headers.get('Reply-To'))

        new_msg.to_addr = parse_email_address_list(parsed.headers.getall('To'))
        new_msg.cc_addr = parse_email_address_list(parsed.headers.getall('Cc'))
        new_msg.bcc_addr = parse_email_address_list(
            parsed.headers.getall('Bcc'))

        new_msg.in_reply_to = parsed.headers.get('In-Reply-To')
        new_msg.message_id_header = parsed.headers.get('Message-Id')

        new_msg.received_date = received_date

        # Optional mailing list headers
        new_msg.mailing_list_headers = parse_ml_headers(parsed.headers)

        # Custom Inbox header
        new_msg.inbox_uid = parsed.headers.get('X-INBOX-ID')

        # In accordance with JWZ (http://www.jwz.org/doc/threading.html)
        new_msg.references = parse_references(
            parsed.headers.get('References', ''),
            parsed.headers.get('In-Reply-To', ''))

        new_msg.size = len(body_string)  # includes headers text

        i = 0  # for walk_index

        # Store all message headers as object with index 0
        headers_part = Part()
        headers_part.namespace_id = account.namespace.id
        headers_part.message = new_msg
        headers_part.walk_index = i
        headers_part.data = json.dumps(parsed.headers.items())
        new_msg.parts.append(headers_part)

        for mimepart in parsed.walk(
                with_self=parsed.content_type.is_singlepart()):
            i += 1
            if mimepart.content_type.is_multipart():
                log.warning("multipart sub-part found! on {}"
                            .format(new_msg.g_msgid))
                continue  # TODO should we store relations?

            new_part = Part()
            new_part.namespace_id = account.namespace.id
            new_part.message = new_msg
            new_part.walk_index = i
            new_part.misc_keyval = mimepart.headers.items()  # everything
            new_part.content_type = mimepart.content_type.value
            new_part.filename = trim_filename(
                mimepart.content_type.params.get('name'),
                log=log)
            # TODO maybe also trim other headers?

            if mimepart.content_disposition[0] is not None:
                value, params = mimepart.content_disposition
                if value not in ['inline', 'attachment']:
                    errmsg = """
    Unknown Content-Disposition on message {0} found in {1}.
    Bad Content-Disposition was: '{2}'
    Parsed Content-Disposition was: '{3}'""".format(
                        mid, folder_name, mimepart.content_disposition)
                    log.error(errmsg)
                    continue
                else:
                    new_part.content_disposition = value
                    if value == 'attachment':
                        new_part.filename = trim_filename(
                            params.get('filename'),
                            log=log)

            if mimepart.body is None:
                data_to_write = ''
            elif new_part.content_type.startswith('text'):
                data_to_write = mimepart.body.encode('utf-8', 'strict')
                # normalize mac/win/unix newlines
                data_to_write = data_to_write \
                    .replace('\r\n', '\n').replace('\r', '\n')
            else:
                data_to_write = mimepart.body
            if data_to_write is None:
                data_to_write = ''

            new_part.content_id = mimepart.headers.get('Content-Id')
            new_part.data = data_to_write
            new_msg.parts.append(new_part)
    except mime.DecodingError:
        # occasionally iconv will fail via maximum recursion depth
        log_decode_error(account.id, folder_name, mid, body_string)
        log.error('DecodeError, msg logged to {0}'.format(
            get_errfilename(account.id, folder_name, mid)))
        return
    except RuntimeError:
        log_decode_error(account.id, folder_name, mid, body_string)
        log.error('RuntimeError<iconv> msg logged to {0}'.format(
            get_errfilename(account.id, folder_name, mid)))
        return

    new_msg.calculate_sanitized_body()
    return new_msg
Example #3
0
def upgrade():

    from inbox.server.config import load_config
    load_config()

    from inbox.server.models import session_scope, Session
    from inbox.server.models.ignition import engine

    from inbox.server.models.tables.base import (Part, Namespace,
                                                 Message, Thread)
    from inbox.sqlalchemy.util import JSON

    print 'Creating table for parts...'
    op.create_table('part',
                    sa.Column('id', sa.Integer(), nullable=False),
                    sa.Column('message_id', sa.Integer(), nullable=True),
                    sa.Column('walk_index', sa.Integer(), nullable=True),
                    sa.Column('content_disposition', sa.Enum(
                        'inline', 'attachment'), nullable=True),
                    sa.Column(
                        'content_id', sa.String(length=255), nullable=True),
                    sa.Column('misc_keyval', JSON(), nullable=True),
                    sa.Column('is_inboxapp_attachment', sa.Boolean(),
                              server_default=sa.sql.expression.false(),
                              nullable=True),
                    sa.ForeignKeyConstraint(
                        ['id'], ['block.id'], ondelete='CASCADE'),
                    sa.ForeignKeyConstraint(
                        ['message_id'], ['message.id'], ondelete='CASCADE'),
                    sa.PrimaryKeyConstraint('id'),
                    sa.UniqueConstraint('message_id', 'walk_index')
                    )

    print 'Reflecting old block table schema'
    Base = declarative_base()
    Base.metadata.reflect(engine)

    class Block_(Base):  # old schema, reflected from database table
        __table__ = Base.metadata.tables['block']

    print 'Adding namespace_id column to blocks ',
    op.add_column(
        u'block', sa.Column('namespace_id', sa.Integer(), nullable=False))

    print 'Migrating from blocks to parts'
    new_parts = []
    with session_scope() as db_session:
        for block in db_session.query(Block_).yield_per(chunk_size):

            # Move relevant fields
            p = Part()
            p.size = block.size
            p.data_sha256 = block.data_sha256
            p.message_id = block.message_id
            p.walk_index = block.walk_index
            p.content_disposition = block.content_disposition
            p.content_id = block.content_id
            p.misc_keyval = block.misc_keyval
            p.is_inboxapp_attachment

            old_namespace = db_session.query(Namespace) \
                .join(Message.thread, Thread.namespace) \
                .filter(Message.id == block.message_id).one()
            p.namespace_id = old_namespace.id

            # Commit after column modifications
            new_parts.append(p)

        print 'Deleting old blocks (now parts)... ',
        db_session.query(Block_).delete()
        db_session.commit()
        print 'Done!'

    print 'Removing `message_id` constraint from block'
    op.drop_constraint('block_ibfk_1', 'block', type_='foreignkey')

    print 'Creating foreign key for block -> namespace on block'
    op.create_foreign_key('block_ibfk_1', 'block', 'namespace',
                          ['namespace_id'], ['id'], ondelete='CASCADE')

    print 'Dropping old block columns which are now in part'
    op.drop_column(u'block', u'walk_index')
    op.drop_column(u'block', u'content_disposition')
    op.drop_column(u'block', u'misc_keyval')
    op.drop_column(u'block', u'content_id')
    op.drop_column(u'block', u'is_inboxapp_attachment')
    op.drop_constraint(u'message_id', 'block', type_='unique')
    op.drop_column(u'block', u'message_id')

    # Note: here we use the regular database session, since the transaction
    # log requires the `namespace` property on objects. We've set the
    # `namespace_id` foreign key, but need to commit the object before the
    # SQLalchemy reference is valid
    no_tx_session = Session(autoflush=True, autocommit=False)
    no_tx_session.add_all(new_parts)
    no_tx_session.commit()

    print 'Done migration blocks to parts!'