Beispiel #1
0
    def reportEmptyPurgeCompleted(self, pid):
        m_container = ManagementContainer.getInstance()
        c = m_container.getDBConnection(ManagementContainer.AM_POOL_NAME)
        s = None
        needsRollback = True
        try:
            try:
                s = c.createStatement()

                # mark the number of messages dispatched
                m_container.getActiveMailboxStoreManager().markDispatched(c, pid, 0)

                # procedure expects partition ID, count of items purged, and duration
                sql = "select msg_purge_mark_purged(%d,0,0)" % pid
                s.execute(sql)
                self.flushPartitionCache(pid)
                c.commit()
                needsRollback = False
            except:
                log("Exception caught trying to call msg_purge_mark_purged()")
        finally:
            if needsRollback:
                c.rollback()
            DbUtils.safeClose(s)
            m_container.safeReturnDBConnection(c, ManagementContainer.AM_POOL_NAME)
Beispiel #2
0
def getNextPartition():
    sql = 'select msg_purge_next_to_compute()'

    mc = ManagementContainer.getInstance()
    conn = mc.getDBConnection(ManagementContainer.AM_POOL_NAME)
    s = None
    rs = None
    needsRollback = True
    try:
        s = conn.createStatement()
        rs = s.executeQuery(sql)
        if rs.next():
            pid = rs.getInt(1)
            conn.commit()
            needsRollback = False
            if pid == 0:
                return None
            else:
                return pid
    finally:
        if (needsRollback):
            conn.rollback()
        DbUtils.safeClose(rs)
        DbUtils.safeClose(s)
        mc.safeReturnDBConnection(conn,ManagementContainer.AM_POOL_NAME)  
    return None
Beispiel #3
0
def getUserDBData(partId,msgId):
    mc = ManagementContainer.getInstance()
    conn = mc.getDBConnection(ManagementContainer.AM_POOL_NAME)

    recipients = []
    sender = 0
    rs = None
    s = None

    try:
        s = conn.createStatement()
        sql = 'select user_id,is_sender from msg_users_%s where m1_message_id = %s' % (partId,msgId)
        rs = s.executeQuery(sql)
        while rs.next():
            id = rs.getInt(1)
            category = RecipientCategory.valueOf(rs.getInt(2))
            if RecipientCategory.SENDER.equals(category):
                sender = id
            else:
                recipients.append(MatchRecipient(id, category))
    finally:
        conn.rollback()
        DbUtils.safeClose(rs)
        DbUtils.safeClose(s)
        mc.safeReturnDBConnection(conn, ManagementContainer.AM_POOL_NAME)

    return (sender,recipients)
Beispiel #4
0
    def getNextPartition(self):
        sql = "select msg_purge_next_to_dispatch(true)"
        # log('getNextPartition() sql ' + sql)

        mc = ManagementContainer.getInstance()
        pm = mc.getPartitionManager()
        conn = mc.getDBConnection(ManagementContainer.AM_POOL_NAME)
        s = None
        rs = None
        needsRollback = True
        try:
            s = conn.createStatement()
            rs = s.executeQuery(sql)
            if rs.next():
                pid = rs.getInt(1)
                # log('getNextPartition() pid = %d' % pid)
                conn.commit()
                needsRollback = False
                if pid == 0:
                    return None
                else:
                    return pm.getPartition(pid)
        finally:
            if needsRollback:
                conn.rollback()
            DbUtils.safeClose(rs)
            DbUtils.safeClose(s)
            mc.safeReturnDBConnection(conn, ManagementContainer.AM_POOL_NAME)
        return None
Beispiel #5
0
 def getRanges(self, partId):
     rc = []
     sql = "select * from generate_purge('msg_purge_todo_%d',%d)" % (partId,partId)
     mc = ManagementContainer.getInstance()
     conn = mc.getPDBConnection(partId)
     s = None
     rs = None
     needsRollback = True
     try:
         s = conn.createStatement()
         rs = s.executeQuery(sql)
 
         while rs.next():
             cmd = rs.getString(1)
             parts = cmd.split(' ')
             rangeStr = parts[-1]
             range = rangeStr.split('-')
             if len(range) != 2:
                 log('Invalid range specification found',rangeStr)
                 continue
             rc.append((long(range[0]),long(range[1])))
         
         conn.commit()
         needsRollback = False
     finally:
         if (needsRollback):
             conn.rollback()
         DbUtils.safeClose(rs)
         DbUtils.safeClose(s)
         mc.returnDBConnection(conn)
             
     return rc
Beispiel #6
0
def setUserPurgeDate(usersToPurge, markPurgeDaysBeforeCurrent):
    mc = ManagementContainer.getInstance()
    connection = mc.getDBConnection(ManagementContainer.POOL_NAME)
    s = connection.createStatement()
    s.executeUpdate("update dat_user_account set purge_date = current_timestamp - '"+str(markPurgeDaysBeforeCurrent)+" days'::interval where object_id in "+QueryUtils.literal(usersToPurge))
    s.close()
    connection.commit()
    mc.safeReturnDBConnection(connection, ManagementContainer.POOL_NAME)
    DbUtils.safeClose(s)
Beispiel #7
0
def updateContentId(oldMessageId, replacementId, partitionId):
    mc = ManagementContainer.getInstance()
    connection = mc.getDBConnection(ManagementContainer.AM_POOL_NAME)
    s = connection.createStatement()
    command = "update dat_repl_messages_" + str(partitionId) + " set content_replacement_id= " + str(replacementId) + " where m1_message_id= " + str(oldMessageId)
    print "sql: " + command
    s.executeUpdate(command)
    connection.commit()
    mc.safeReturnDBConnection(connection, ManagementContainer.AM_POOL_NAME)
    DbUtils.safeClose(s)
Beispiel #8
0
def updatePassword(email, hashedPassword):
    mc = ManagementContainer.getInstance()
    connection = mc.getDBConnection(ManagementContainer.POOL_NAME)
    s = connection.createStatement()
    command = "update dat_user_account set hashed_password = \'" + str(hashedPassword) + "\' where primary_email = \'" + email + "\'"
    print "EXEC: " + command
    s.executeUpdate(command)
    connection.commit()
    mc.safeReturnDBConnection(connection, ManagementContainer.POOL_NAME)
    DbUtils.safeClose(s)
Beispiel #9
0
def getPartitionId(custId,archiveId):
    mc = ManagementContainer.getInstance()
    connection = mc.getDBConnection(ManagementContainer.AM_POOL_NAME)
    s = connection.createStatement()
    command = "select partition_id from archive_chunk where customer_id=" + custId + " and archive_id=" + archiveId
    print "EXEC: " + command
    resultSet = s.executeQuery(command)
    resultSet.next()
    partition_id = resultSet.getInt(1)
    mc.safeReturnDBConnection(connection, ManagementContainer.AM_POOL_NAME)
    DbUtils.safeClose(s)
    return partition_id
Beispiel #10
0
def getArchiveMessages(partitionId,archiveId):
    mc = ManagementContainer.getInstance()
    connection = mc.getDBConnection(ManagementContainer.AM_POOL_NAME)
    s = connection.createStatement()
    command = "select count(*) from archive_chunk_message_" + str(partitionId) + " where archive_id=" + archiveId
    print "EXEC: " + command
    resultSet = s.executeQuery(command)
    resultSet.next()
    count = resultSet.getInt(1)
    mc.safeReturnDBConnection(connection, ManagementContainer.AM_POOL_NAME)
    DbUtils.safeClose(s)
    return count
Beispiel #11
0
def removePurgeHold(mc,custId):
    conn = mc.getDBConnection(ManagementContainer.AM_POOL_NAME)
    stmt = None
    needsRollback = True
    try:
        stmt = conn.createStatement()
        stmt.executeUpdate('delete from purge_holds where customer_id = ' + str(custId))
        conn.commit()
        needsRollback = False
    finally:
        if (needsRollback):
            conn.rollback()
        DbUtils.safeClose(stmt)
        mc.safeReturnDBConnection(conn,ManagementContainer.AM_POOL_NAME)
Beispiel #12
0
def checkPassword(email, pword):
    mc = ManagementContainer.getInstance()
    connection = mc.getDBConnection(ManagementContainer.POOL_NAME)
    s = connection.createStatement()
    command = "select hashed_password from dat_user_account where primary_email = \'" + email + "\'"
    print "EXEC: " + command
    resultSet = s.executeQuery(command)
    resultSet.next()
    hashedPassword = resultSet.getString(1)
    mc.safeReturnDBConnection(connection, ManagementContainer.POOL_NAME)
    DbUtils.safeClose(s)
    authMgr = mc.getAuthenticationManager()
    if not authMgr.checkPassword(pword, hashedPassword):
        print "Hashed passwords do not match"
        sys.exit(1)
Beispiel #13
0
def getLocId(locName):
    conn = ManagementContainer.getInstance().getDBConnection(ManagementContainer.AM_POOL_NAME)
    s = None
    rs = None
    try:
        s = conn.createStatement()
        q = "select location_id from dat_cluster_locations where location = '%s'" % str(locName)
        rs = s.executeQuery(q)
        if rs.next():
            return rs.getInt(1)
        return None
    finally:
        conn.rollback()
        DbUtils.safeClose(rs)
        DbUtils.safeClose(s)
        ManagementContainer.getInstance().safeReturnDBConnection(conn, ManagementContainer.AM_POOL_NAME)
Beispiel #14
0
def initPurge():
        sql = "select msg_purge_initialize()"
        log('initPurge:', sql)
        mc = ManagementContainer.getInstance()
        conn = mc.getDBConnection(ManagementContainer.AM_POOL_NAME)
        s = None
        needsRollback = True
        try:
            s = conn.createStatement()
            s.executeQuery(sql)
            conn.commit()
            needsRollback = False
        finally:
            if (needsRollback):
                conn.rollback()
            DbUtils.safeClose(s)
            mc.safeReturnDBConnection(conn,ManagementContainer.AM_POOL_NAME)
Beispiel #15
0
def createPurgeHold(custID, days):
    sql = "insert into purge_holds values (" + str(custID) + ",current_timestamp + '" + str(days) + " days'::interval);"
    stmt = None;    
    cont = ManagementContainer.getInstance()
    c = cont.getDBConnection(ManagementContainer.AM_POOL_NAME)
    rollback = True
    try:
        stmt = c.createStatement()
        stmt.executeUpdate(sql);
        print "Added purge hold for customer", custID, "for", days, "days."
        c.commit();
        rollback = False
    finally:
        DbUtils.safeClose(stmt)
        if ( rollback ):
            c.rollback()
        cont.safeReturnDBConnection(c,ManagementContainer.AM_POOL_NAME)
Beispiel #16
0
 def shouldChunk(self, partId, daysSinceLastPurge):
     sql = "select reltuples from pg_class where relname = 'dat_repl_messages_%d'" % partId
     mc = ManagementContainer.getInstance()
     conn = mc.getPDBConnection(partId)
     s = None
     rs = None
     tuples = 0
     try:
         s = conn.createStatement()
         rs = s.executeQuery(sql)
         if rs.next():
             tuples = rs.getLong(1)
     finally:
         conn.rollback()
         DbUtils.safeClose(rs)
         DbUtils.safeClose(s)
         mc.returnDBConnection(conn)
     
     return (daysSinceLastPurge * 0.01 * tuples) >= self.maxTupleThreshold
Beispiel #17
0
def getDBData(partId,msgId):
    mc = ManagementContainer.getInstance()
    conn = mc.getDBConnection(ManagementContainer.AM_POOL_NAME)
    s = None
    rs = None
    try:
        s = conn.createStatement()
        sql = 'select m1_message_id,customer_id,index_status,subject from dat_repl_messages_%s where m1_message_id = %s' % (partId,msgId)
        rs = s.executeQuery(sql)
        if rs.next():
            return (rs.getLong(1),rs.getInt(2),rs.getInt(3),rs.getString(4))
    finally:
       conn.rollback()
       DbUtils.safeClose(rs)
       DbUtils.safeClose(s)
       mc.safeReturnDBConnection(conn, ManagementContainer.AM_POOL_NAME)

    #print time.asctime(),'Cannot find info for %s,%s' % (partId,msgId)
    return (long(msgId),0,0,None)
Beispiel #18
0
def alterEmailFile(partitionId, messageId, extension, append=True):
    mc = ManagementContainer.getInstance()
    connection = mc.getDBConnection(ManagementContainer.AM_POOL_NAME)
    s = connection.createStatement()
    command = "select storage_location,customer_id from dat_repl_messages_" + str(partitionId) + " where m1_message_id=" + str(messageId)
    print "sql: " + command
    resultSet = s.executeQuery(command)
    resultSet.next()
    storageLocation = resultSet.getString(1)
    customerId = resultSet.getString(2)
    mc.safeReturnDBConnection(connection, ManagementContainer.AM_POOL_NAME)
    DbUtils.safeClose(s)
    storageLocation = "/ems/bigdisk/storagep" + partitionId + "/" + customerId + "/" + storageLocation
    if append is True:
        command = "mv " + storageLocation + " " + storageLocation + extension
        print "EXEC: " + command
        os.system(command)
    else:
        command = "mv " + storageLocation + extension + " " + storageLocation
        print "EXEC: " + command
        os.system(command)
Beispiel #19
0
def moveMessage(mc,clusLocId,partId,messageId,customerId):
    conn = mc.getDBConnection(ManagementContainer.AM_POOL_NAME)
    doRefeed = False
    s = None
    try:
        s = conn.createStatement()
        sql = 'update dat_repl_messages_%d set index_status = %d where m1_message_id = %d' % (int(partId),int(clusLocId),long(messageId))
        try:
            s.executeUpdate(sql)
            conn.commit()
            doRefeed = True
        except:
            conn.rollback()
            print 'Exception (transaction aborted)', sql, sys.exc_info(), traceback.print_exc(file=sys.stderr)
    finally:
        DbUtils.safeClose(s)
        mc.safeReturnDBConnection(conn, ManagementContainer.AM_POOL_NAME)

    if doRefeed:
        senderId,recipIds = getUserDBData(partId,messageId)
        refeedMessage(mc,clusLocId,partId,messageId,customerId,senderId,recipIds)
Beispiel #20
0
def expireWinAuthPassword(email):
    mc = ManagementContainer.getInstance()
    connection = mc.getDBConnection(ManagementContainer.POOL_NAME)
    s = connection.createStatement()
    command = "update dat_user_account set password_expire_time=creation_date where primary_email = \'" + email + "\'"
    print "EXEC: " + command
    s.executeUpdate(command)
    connection.commit()
    command = "/opt/ems/bin/emscmd clean-expired-passwords"
    os.system(command)
    command = "select hashed_password,hashed_imap_password from dat_user_account where primary_email = \'" + email + "\'"
    print "EXEC: " + command
    resultSet = s.executeQuery(command)
    resultSet.next()
    hashedPassword = resultSet.getString(1)
    hashedImapPassword = resultSet.getString(2)
    mc.safeReturnDBConnection(connection, ManagementContainer.POOL_NAME)
    DbUtils.safeClose(s)
    if not hashedPassword is None or not hashedImapPassword is None:
        print "Hashed password=\"", hashedPassword, "\" Hashed Imap Password=\"", hashedImapPassword, "\""
        sys.exit(1)
Beispiel #21
0
def updateCustomerNames():
    emails = ["*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**"]
    chinese = "中国的"
    greek = "ελληνικά"
    modi = "મોદી"
    arabic = "العربية"
    japanese = "日本人"
    hebrew = "עברית"
    names = [greek,hebrew,arabic,modi,chinese,japanese]
    mc = ManagementContainer.getInstance()
    connection = mc.getDBConnection(ManagementContainer.POOL_NAME)
    s = connection.createStatement()
    count = 0
    for name in names:
        command = "update dat_user_account set display_name = \'" + name + "\' where primary_email = \'" + emails[count] + "\'"
        print "EXEC: " + command
        command = String("update dat_user_account set display_name = \'" + name + "\' where primary_email = \'" + emails[count] + "\'", "utf-8")
        s.executeUpdate(command)
        count +=1
    connection.commit()
    mc.safeReturnDBConnection(connection, ManagementContainer.POOL_NAME)
    DbUtils.safeClose(s)
Beispiel #22
0
def preCalcPartition(id):
    log('preCalcPartition(%d)' % id)
    mc = ManagementContainer.getInstance()
    pm = mc.getPartitionManager()

    sql = 'select msg_purge_compute(%d)' % id
    log('preCalcPartition() invoking %s' % sql)
    conn = mc.getPDBConnection(id)
    s = None
    needsRollback = True
    try:
        s = conn.createStatement()
        s.executeQuery(sql)
        conn.commit()
        needsRollback = False
    finally:
        if (needsRollback):
            conn.rollback()
        DbUtils.safeClose(s)
        mc.returnDBConnection(conn)
        
    log('preCalcPartition(%d) done' % id)
Beispiel #23
0
    def listPartitions(self):
        sql = 'select id from dat_active_mailbox_partition order by purge_order'

        mc = ManagementContainer.getInstance()
        pm = mc.getPartitionManager()
        conn = mc.getDBConnection(ManagementContainer.AM_POOL_NAME)
        s = None
        rs = None
        rc = []
        try:
            s = conn.createStatement()
            rs = s.executeQuery(sql)
            while rs.next():
                pid = rs.getInt(1)
                rc.append(pm.getPartition(pid))
        finally:
            conn.rollback()
            DbUtils.safeClose(rs)
            DbUtils.safeClose(s)
            mc.safeReturnDBConnection(conn,ManagementContainer.AM_POOL_NAME)

        return rc       
Beispiel #24
0
def markMessageMissing(messageId,pid,contentSubject):
    mc = ManagementContainer.getInstance()
    cr = mc.getContentReplacementDAO()
    connection = mc.getDBConnection(mc.AM_POOL_NAME)
    s = connection.createStatement()

    #Make sure a content replacement for this content exists and if not create one
    command = "/opt/ems/bin/emscmd add-content-replacement -c \"" + contentSubject + "\" -n \"" + "test Notes" + "\""
    print "EXEC: " + command
    process = os.popen(command)
    output = process.read()
    process.close()
    print "OUT: " + output
    contentId = re.search('m_id=(.+?),m_notes', output).group(1)

    print "CONTENT ID " +str(contentId)

    #Mark the message as missing in the db
    updateRepl = "update dat_repl_messages_" + str(pid) + " set content_replacement_id = " + str(contentId) + " where m1_message_id = " + str(messageId)
    updateKeyMap = "update dat_repl_message_key_map set requires_restore = true where m1_message_id = " + str(messageId)


    #Grab the storage location and customer id of the message
    results = getStorageLocation(pid,messageId,s)

    s.executeUpdate(updateRepl)
    s.executeUpdate(updateKeyMap)
    connection.commit()
    mc.safeReturnDBConnection(connection,mc.AM_POOL_NAME)
    DbUtils.safeClose(s)
    connection.close()

    storageLocation = results[1]
    customerId = results[0]
    directory = "/ems/bigdisk/storagep"+str(pid)+"/"+str(customerId)+"/"
    pathname = os.path.abspath(os.path.join(directory,storageLocation))
    #add .bak to the message content file so the system thinks that it is missing
    os.rename(pathname,pathname+".bak")
Beispiel #25
0
            needsRollback = True
            sql = 'select msg_purge_next_to_precompute(' + str(days) + ',' + str(id) + ')'
            rs = s.executeQuery(sql)
            if rs.next():
                id = rs.getInt(1)
            if id > 0:
                partIdList.append(id)
                conn.commit()
                needsRollback = False
            else:
                done = True
        
    finally:
        if (needsRollback):
            conn.rollback()
        DbUtils.safeClose(rs)
        DbUtils.safeClose(s)
        mc.safeReturnDBConnection(conn,ManagementContainer.AM_POOL_NAME)

    # build ranges by partition/shard
    # todo - execute shard updates in parallel
    for partId in partIdList:
        conn = mc.getPDBConnection(partId)
        s = None
        needsRollback = True
        try:
            s = conn.createStatement()
            sql = 'select msg_purge_partition(' + str(partId) + ', false,false)'
            s.executeQuery(sql)
            sql = "select ddl_table_create('msg_purge_todo_%d','create table msg_purge_todo_%d as select m1_message_id from msg_purge_%d where policy_delete_date < current_timestamp', true)"
            sql = sql % (partId,partId,partId)
            s = conn.createStatement()
            done = False
            id = -1
            for msg_id in out:
                sql = 'select partition_id from dat_repl_message_key_map where m1_message_id=' + msg_id
                rs = s.executeQuery(sql)
                if rs.next():
                    id = rs.getInt(1)
                    if id > 0:
                        print "Message id ", msg_id.strip(), "PartitionId = ", id
                        data.update({msg_id: id})
                        #partIdList.append(id)
                        conn.commit()
                    else:
                        done = True

        finally:
            conn.rollback()
            DbUtils.safeClose(rs)
            DbUtils.safeClose(s)
            mc.safeReturnDBConnection(conn, ManagementContainer.AM_POOL_NAME)

            for key in data:
                print "executing command for msgId ", key.strip(), "partId ", data[key]
                optCmd = "/opt/ems/bin/amp purge-v2 -P -i " + str(
                    data[key]) + " -w 'md.m1_message_id=" + key.strip() + "\' " + "2>/dev/null"
                print optCmd
                os.system(optCmd)

        sys.exit(0)