Пример #1
0
def project_msg_banner(user, project_msg, action):
    try:
        msg = ""
        thistag = ""
        logger.debug("Function: %s()" % sys._getframe().f_code.co_name)
        for pk, status_list in project_msg.iteritems():
            url = urlresolvers.reverse("dm_log", args=(pk,))
            report = Results.objects.get(id=pk)
            msg += "(%s) %s " % (report.resultsName, action.title())
            for category, status in status_list.iteritems():
                msg += " %s, " % str(category)
                msg += status
                grpstatus = status
                thistag = "%s_%s_%s" % (str(pk), action, slugify(category))
            msg += " <a href='%s'  data-toggle='modal' data-target='#modal_report_log'>View Report Log</a></br>" % (url)
            logger.debug("MESSAGE: %s" % msg)

        # If message already exists (ie, scheduled task) delete it.
        Message.objects.filter(tags=thistag).delete()

        if len(project_msg) > 1:
            thistag = "%s_%s_%s" % ("project", action, slugify(category))

        if grpstatus == "scheduled":
            func = Message.info
        elif grpstatus == "success":
            func = Message.success
        else:
            func = Message.error
    except:
        func = Message.error
        logger.exception(traceback.format_exc())
    return func(msg, route=user, tags=thistag)
Пример #2
0
def show_keepers():

    for dmtype in FILESET_TYPES:

        # filename stem
        storename = "%s_%s_keepers" % (slugify(dmtype), gethostname())

        # All SigProc Stat objects
        sigprocstats = models.DMFileStat.objects.filter(dmfileset__type=dmtype).order_by('created')
        print "All %s objects count: %d" % (dmtype, sigprocstats.count())

        # Limit to objects marked keep
        sigprocstats = sigprocstats.filter(result__experiment__storage_options = 'KI')
        print "All %s objects Local marked Keep count: %d" % (dmtype, sigprocstats.count())

        # Limit to objects with files on filesystem
        sigprocstats = sigprocstats.filter(action_state='L')
        print "All %s objects Local count: %d" % (dmtype, sigprocstats.count())

        _write_csv(sigprocstats, storename)
        _write_excel(sigprocstats, storename)
Пример #3
0
def show_keepers():

    for dmtype in FILESET_TYPES:

        # filename stem
        storename = "%s_%s_keepers" % (slugify(dmtype), gethostname())

        # All SigProc Stat objects
        sigprocstats = models.DMFileStat.objects.filter(
            dmfileset__type=dmtype).order_by('created')
        print "All %s objects count: %d" % (dmtype, sigprocstats.count())

        # Limit to objects marked keep
        sigprocstats = sigprocstats.filter(
            result__experiment__storage_options='KI')
        print "All %s objects Local marked Keep count: %d" % (
            dmtype, sigprocstats.count())

        # Limit to objects with files on filesystem
        sigprocstats = sigprocstats.filter(action_state='L')
        print "All %s objects Local count: %d" % (dmtype, sigprocstats.count())

        _write_csv(sigprocstats, storename)
        _write_excel(sigprocstats, storename)
Пример #4
0
def fileserver_space_check():
    '''For each file server, compare disk usage to backup threshold value.
    If disk usage exceeds threshold, launch celery task to delete/archive
    raw data directories.
    '''
    logger = get_task_logger('data_management')

    # Get GlobalConfig object in order to access auto-acknowledge bit
    gc = GlobalConfig.get()
    auto_acknowledge = gc.auto_archive_ack
    auto_action_enabled = gc.auto_archive_enable

    # Get list of File Server objects
    file_servers = FileServer.objects.all().order_by('pk').values()

    # Get list of Report Storage objects
    report_storages = ReportStorage.objects.all().order_by('pk').values()

    # dict of fileset cateogries each with list of partition ids that can be acted upon.
    category_list = {}
    #-------------------------------------------------
    # DELETE action only happens if threshold reached
    #-------------------------------------------------
    for dmfileset in DMFileSet.objects.filter(version=settings.RELVERSION).filter(auto_action='DEL').values():

        cat_name = slugify(dmfileset['type'])
        category_list[cat_name] = {
            'dmfileset':dmfileset,
            'devlist':[],
            'partitions':[],
        }

        for partition in _partitions(file_servers,report_storages):

            if partition['diskusage'] >= dmfileset['auto_trigger_usage']:

                logger.info("%s %s %.2f%% exceeds %s %.0f%%" % (
                    hex(partition['devid']),
                    partition['path'],
                    partition['diskusage'],
                    dmfileset['type'],
                    dmfileset['auto_trigger_usage']))

                category_list[cat_name]['devlist'].append(partition['devid'])
                category_list[cat_name]['partitions'].append(partition)

            else:

                logger.info("%s %s %.2f%% below %s %.0f%%" % (
                    hex(partition['devid']),
                    partition['path'],
                    partition['diskusage'],
                    dmfileset['type'],
                    dmfileset['auto_trigger_usage']))

        # uniquify the deviceid list
        category_list[cat_name]['devlist'] = list(set(category_list[cat_name]['devlist']))

    #-------------------------------------------------------------------------------
    #ARCHIVE action happens as soon as grace period has expired (no threshold check)
    #-------------------------------------------------------------------------------
    for dmfileset in DMFileSet.objects.filter(version=settings.RELVERSION).filter(auto_action='ARC').values():

        cat_name = slugify(dmfileset['type'])
        category_list[cat_name] = {
            'dmfileset':dmfileset,
            'devlist':[],
            'partitions':[],
        }

        for partition in _partitions(file_servers,report_storages):
            logger.debug("%s %s" %( partition['path'],hex(partition['devid'])))
            category_list[cat_name]['devlist'].append(partition['devid'])
            category_list[cat_name]['partitions'].append(partition)

        # uniquify the deviceid list
        category_list[cat_name]['devlist'] = list(set(category_list[cat_name]['devlist']))


    #-------------------------------------------------------------
    # Action loop - for each category, launch action per deviceid
    #-------------------------------------------------------------
    if not auto_action_enabled:
        logger.info("Data management auto-action is disabled.")

    # update any DMFileStats that are in use by active analysis jobs
    try:
        update_files_in_use()
    except:
        logger.error('Unable to update active DMFileStats')
        logger.error(traceback.format_exc())

    # Checks for manual export and archive requests.
    manage_manual_action.delay()

    for category,dict in category_list.iteritems():
        for deviceid in dict['devlist']:
            pathlist = [item['path'] for item in dict['partitions'] if item['devid'] == deviceid]
            manage_data.delay(deviceid, dict['dmfileset'], pathlist, auto_acknowledge, auto_action_enabled)

    return
Пример #5
0
def manage_data(deviceid, dmfileset, pathlist, auto_acknowledge_enabled, auto_action_enabled):
    logger.debug("manage_data: %s %s %s" % (dmfileset['auto_action'], hex(deviceid),dmfileset['type']))

    def getfirstnotpreserved(dmfilestats, action):
        '''QuerySet of DMFileStat objects.  Returns first instance of an object
        with preserve_data set to False and passes the action_validation test'''
        logger.debug("Function: %s()" % sys._getframe().f_code.co_name)
        logger.debug("Looking at %d dmfilestat objects" % dmfilestats.count())
        for archiveme in dmfilestats:
            if not archiveme.getpreserved():
                try:
                    dmactions.action_validation(archiveme,action)
                    return archiveme
                except:
                    logger.debug("%s Failed action_validation.  Try next fileset" % archiveme.result.resultsName)
            else:
                logger.debug("Skipped a preserved fileset: %s" % archiveme.result.resultsName)

        logger.info("%d filestat objects are preserved." % dmfilestats.count())
        raise DMExceptions.NoDMFileStat("NONE FOUND")

    try:
        #logger.debug("manage_data lock for %s (%d)" % (dmfileset['type'], os.getpid()))
        #Create lock file to prevent more than one celery task for each process_type and partition
        uniqid = "%s_%s" % (hex(deviceid),slugify(dmfileset['type']))
        lockfile = '/var/run/celery/%s.lock' % (uniqid)
        applock = ApplicationLock(lockfile)

        if not(applock.lock()):
            logger.info("failed to acquire lock file: %s" % uniqid)
            return

        logger.info("Worker PID %d lockfile created %s" % (os.getpid(),uniqid))

    except Exception as e:
        logger.exception(e)

    try:
        #---------------------------------------------------------------------------
        # Database object filtering
        #---------------------------------------------------------------------------
        actiondmfilestat = None
        user_comment = "Auto Action"
        # Order by incrementing pk.  This puts them in chronological order.
        # Select DMFileStat objects of category DMFileSet.type (1/4th of all objects)
        dmfilestats = DMFileStat.objects.filter(dmfileset__type=dmfileset['type']).order_by('created')
        tot_obj = dmfilestats.count()

        # Select objects not yet processed
        dmfilestats = dmfilestats.filter(action_state__in=['L','S','N','A'])
        tot_act = dmfilestats.count()
        logger.info("(%s)Total %s: %d Active: %d" %(hex(deviceid),dmfileset['type'],tot_obj,tot_act))

        # Select objects that are old enough
        threshdate = datetime.now(pytz.UTC) - timedelta(days=dmfileset['auto_trigger_age'])
        dmfilestats = dmfilestats.filter(created__lt=threshdate)

        # Select objects stored on the deviceid
        query = Q()
        for path in pathlist:
            if dmfileset['type'] == dmactions_types.SIG:
                dmfilestats = dmfilestats.filter(result__experiment__expDir__startswith=path)
            else:
                dmfilestats = dmfilestats.filter(result__reportstorage__dirPath__startswith=path)

        #    query |= Q(result__experiment__expDir__startswith=path) | Q(result__reportstorage__dirPath__startswith=path)
        #dmfilestats = dmfilestats.filter(query)


        #---------------------------------------------------------------------------
        # Archive
        #---------------------------------------------------------------------------
        if dmfileset['auto_action'] == 'ARC':
            '''
            Rules:
            1) archive a fileset as soon as age threshold reached, regardless of disk threshold
            2) select oldest fileset
            3) select unprocessed fileset
            4) Archiving does not require 'S' -> 'N' -> 'A' progression before action
            5) Do not include filesets marked 'E' in auto-action - can get stuck on that fileset forever
            '''
            logger.info("(%s)Exceed %d days. Eligible to archive: %d" %(hex(deviceid),dmfileset['auto_trigger_age'],dmfilestats.count()))
            actiondmfilestat = None
            #Bail out if disabled
            if auto_action_enabled != True:
                logger.info("Data management auto-action is disabled.")
                return

            # Select first object stored on the deviceid
            try:
                actiondmfilestat = getfirstnotpreserved(dmfilestats, dmactions.ARCHIVE)
                logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName))
            except DMExceptions.NoDMFileStat:
                logger.debug("No filesets to archive on this device: %s" % hex(deviceid))
            except:
                logger.error(traceback.format_exc())
            else:
                archive_action('dm_agent', user_comment, actiondmfilestat)

        #---------------------------------------------------------------------------
        # Delete
        #---------------------------------------------------------------------------
        elif dmfileset['auto_action'] == 'DEL':
            '''
            Rules: If auto-acknowledge is True:
            promote to Acknowledged and delete when age and disk threshold reached.
            If auto-acknowledge is False:
                If fileset type is SIG:
                    'S' -> 'N' -> 'A' progression
                Else:
                    promote to 'A'
                delete an 'A' fileset
            '''
            logger.info("(%s)Exceed %d days. Eligible to delete: %d" %(hex(deviceid),dmfileset['auto_trigger_age'],dmfilestats.count()))
            if auto_acknowledge_enabled:
                if dmfileset['type'] == dmactions_types.SIG:
                    logger.debug("Sig Proc Input Files auto acknowledge enabled")
                '''
                Do not require 'S' -> 'N' -> 'A' progression before action; mark 'A' and process
                '''
                A_list = dmfilestats.filter(action_state='A')
                if A_list.count() > 0:
                    # there are filesets acknowledged and ready to be deleted.  Covers situation where user has
                    #already acknowledged but recently enabled auto-acknowledge as well.
                    #deleteme = A_list[0]
                    try:
                        actiondmfilestat = getfirstnotpreserved(A_list, dmactions.DELETE)
                        logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName))
                    except DMExceptions.NoDMFileStat:
                        logger.info("(%s) No filesets to delete on this device" % hex(deviceid))
                    except:
                        logger.error(traceback.format_exc())

                if actiondmfilestat == None:
                    # Select oldest fileset regardless if its 'L','S','N','A'.  This covers situation where user
                    # recently enabled auto-acknowledge
                    try:
                        actiondmfilestat = getfirstnotpreserved(dmfilestats, dmactions.DELETE)
                        logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName))
                    except DMExceptions.NoDMFileStat:
                        logger.info("(%s) No filesets to delete on this device" % hex(deviceid))
                    except:
                        logger.error(traceback.format_exc())
            else:
                if dmfileset['type'] == dmactions_types.SIG:
                    logger.debug("Sig Proc Input Files auto acknowledge disabled")
                    '''
                    Need to select # of filesets and mark 'S'
                    Need to find 'S' filesets and notify (set to 'N')
                    Set queue length to 5
                    sum(S,N,A) == queue length
                    '''
                    #Get email recipient
                    try:
                        recipient = User.objects.get(username='******').email
                        recipient = recipient.replace(',',' ').replace(';',' ').split()
                    except:
                        recipient = None

                    # Need to select Experiments to process with appropriate dmfilestats action states.
                    exps = Experiment.objects.exclude(storage_options='KI').exclude(expDir='').order_by('date')
                    L_list = exps.filter(results_set__dmfilestat__in = dmfilestats.filter(action_state='L')).distinct()
                    S_list = exps.filter(results_set__dmfilestat__in = dmfilestats.filter(action_state='S')).distinct()
                    N_list = exps.filter(results_set__dmfilestat__in = dmfilestats.filter(action_state='N')).distinct()
                    A_list = exps.filter(results_set__dmfilestat__in = dmfilestats.filter(action_state='A')).distinct()

                    logger.info("Experiments (Keep=False) L:%d S:%d N:%d A:%d" % (L_list.count(),S_list.count(),N_list.count(),A_list.count()))
                    queue_length = 5
                    to_select = queue_length - (A_list.count() + N_list.count() + S_list.count())
                    if to_select > 0:
                        # Mark to_select number of oldest Experiments, from 'L', to 'S'
                        promoted = dmfilestats.filter(result__experiment__id__in=list(L_list[:to_select].values_list('id',flat=True)))
                        if auto_action_enabled:
                            promoted.update(action_state = 'S')
                            for dmfilestat in promoted:
                                EventLog.objects.add_entry(dmfilestat.result, "Signal Processing Input Selected for Deletion", username='******')

                    # Get updated list of Selected items
                    selected = dmfilestats.filter(action_state='S')

                    # Send Selected items to be notified
                    if selected.count() > 0 and auto_action_enabled:
                        logger.debug("notify recipient %s" % (recipient))
                        if notify([dmfilestat.result.resultsName for dmfilestat in selected],recipient):
                            selected.update(action_state = 'N')
                            for dmfilestat in selected:
                                EventLog.objects.add_entry(dmfilestat.result, "Notification for Deletion Sent", username='******')

                    try:
                        actiondmfilestat = getfirstnotpreserved(dmfilestats.filter(action_state='A'), dmactions.DELETE)
                        logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName))
                    except DMExceptions.NoDMFileStat:
                        logger.info("(%s) No filesets to delete on this device" % hex(deviceid))
                    except:
                        logger.error(traceback.format_exc())

                else:
                    try:
                        actiondmfilestat = getfirstnotpreserved(dmfilestats, dmactions.DELETE)
                        logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName))
                    except DMExceptions.NoDMFileStat:
                        logger.info("(%s) No filesets to delete on this device" % hex(deviceid))
                    except:
                        logger.error(traceback.format_exc())

            #Bail out if disabled
            if auto_action_enabled != True:
                logger.info("Data management auto-action is disabled.")
                return

            if actiondmfilestat is not None:
                delete_action('dm_agent', user_comment, actiondmfilestat)

        else:
            logger.error("Unknown or unhandled action: %s" % dmfileset['auto_action'])

    except (DMExceptions.FilePermission,
            DMExceptions.InsufficientDiskSpace,
            DMExceptions.MediaNotSet,
            DMExceptions.MediaNotAvailable,
            DMExceptions.FilesInUse) as e:
        message  = Message.objects.filter(tags__contains=e.tag)
        if not message:
            Message.error(e.message,tags=e.tag)
            #TODO: TS-6525: This logentry will repeat for an Archive action every 30 seconds until the cause of the exception is fixed.
            #at least, while the message banner is raised, suppress additional Log Entries.
            EventLog.objects.add_entry(actiondmfilestat.result,"%s - %s" % (dmfileset['type'], e.message),username='******')
    except DMExceptions.SrcDirDoesNotExist as e:
        if actiondmfilestat.dmfileset.type == dmactions_types.SIG:
            msg = "Src Dir not found: %s. Setting action_state to Deleted" % e.message
            EventLog.objects.add_entry(actiondmfilestat.result,msg,username='******')
            actiondmfilestat.setactionstate('DD')
            logger.info(msg)
    except Exception as inst:
        msg = ''
        if actiondmfilestat:
            msg = "Auto-action error on %s " % actiondmfilestat.result.resultsName
        msg += " Error: %s" % str(inst)
        logger.exception("%s - %s" % (dmfileset['type'], msg))

        if actiondmfilestat:
            EventLog.objects.add_entry(actiondmfilestat.result,"%s - %s" % (dmfileset['type'], msg),username='******')
    finally:
        applock.unlock()
        #logger.debug("(%05d,%05d) Release lock file" % (os.getppid(),os.getpid()))

    return
Пример #6
0
def fileserver_space_check():
    '''For each file server, compare disk usage to backup threshold value.
    If disk usage exceeds threshold, launch celery task to delete/archive
    raw data directories.
    '''
    # Get GlobalConfig object in order to access auto-acknowledge bit
    gc = GlobalConfig.get()
    auto_acknowledge = gc.auto_archive_ack
    auto_action_enabled = gc.auto_archive_enable

    # Get list of File Server objects
    file_servers = FileServer.objects.all().order_by('pk').values()

    # Get list of Report Storage objects
    report_storages = ReportStorage.objects.all().order_by('pk').values()

    # dict of fileset cateogries each with list of partition ids that can be acted upon.
    category_list = {}
    #-------------------------------------------------
    # DELETE action only happens if threshold reached
    #-------------------------------------------------
    for dmfileset in DMFileSet.objects.filter(
            version=settings.RELVERSION).filter(auto_action='DEL').values():

        cat_name = slugify(dmfileset['type'])
        category_list[cat_name] = {
            'dmfileset': dmfileset,
            'devlist': [],
            'partitions': [],
        }

        for partition in _partitions(file_servers, report_storages):

            if partition['diskusage'] >= dmfileset['auto_trigger_usage']:

                logger.info("%s %s %.2f%% exceeds %s %.0f%%" %
                            (hex(partition['devid']), partition['path'],
                             partition['diskusage'], dmfileset['type'],
                             dmfileset['auto_trigger_usage']))

                category_list[cat_name]['devlist'].append(partition['devid'])
                category_list[cat_name]['partitions'].append(partition)

            else:

                logger.info("%s %s %.2f%% below %s %.0f%%" %
                            (hex(partition['devid']), partition['path'],
                             partition['diskusage'], dmfileset['type'],
                             dmfileset['auto_trigger_usage']))

        # uniquify the deviceid list
        category_list[cat_name]['devlist'] = list(
            set(category_list[cat_name]['devlist']))

    #-------------------------------------------------------------------------------
    #ARCHIVE action happens as soon as grace period has expired (no threshold check)
    #-------------------------------------------------------------------------------
    for dmfileset in DMFileSet.objects.filter(
            version=settings.RELVERSION).filter(auto_action='ARC').values():

        cat_name = slugify(dmfileset['type'])
        category_list[cat_name] = {
            'dmfileset': dmfileset,
            'devlist': [],
            'partitions': [],
        }

        for partition in _partitions(file_servers, report_storages):
            logger.debug("%s %s" %
                         (partition['path'], hex(partition['devid'])))
            category_list[cat_name]['devlist'].append(partition['devid'])
            category_list[cat_name]['partitions'].append(partition)

        # uniquify the deviceid list
        category_list[cat_name]['devlist'] = list(
            set(category_list[cat_name]['devlist']))

    #-------------------------------------------------------------
    # Action loop - for each category, launch action per deviceid
    #-------------------------------------------------------------
    if not auto_action_enabled:
        logger.info("Data management auto-action is disabled.")

    # update any DMFileStats that are in use by active analysis jobs
    try:
        update_files_in_use()
    except:
        logger.error('Unable to update active DMFileStats')
        logger.error(traceback.format_exc())

    # Checks for manual export and archive requests.
    manage_manual_action.delay()

    for category, dict in category_list.iteritems():
        for deviceid in dict['devlist']:
            pathlist = [
                item['path'] for item in dict['partitions']
                if item['devid'] == deviceid
            ]
            async_task_result = manage_data.delay(deviceid, dict['dmfileset'],
                                                  pathlist, auto_acknowledge,
                                                  auto_action_enabled)

    return
Пример #7
0
def manage_data(deviceid, dmfileset, pathlist, auto_acknowledge_enabled,
                auto_action_enabled):
    logger.debug("manage_data: %s %s %s" %
                 (dmfileset['auto_action'], hex(deviceid), dmfileset['type']))

    def getfirstnotpreserved(dmfilestats, action, threshdate):
        '''QuerySet of DMFileStat objects.  Returns first instance of an object
        with preserve_data set to False and passes the action_validation test'''
        logger.debug("Function: %s()" % sys._getframe().f_code.co_name)
        logger.debug("Looking at %d dmfilestat objects" % dmfilestats.count())
        for archiveme in dmfilestats:
            if not archiveme.getpreserved():
                try:
                    dmactions.action_validation(archiveme, action)
                    return archiveme
                except (DMExceptions.FilesInUse, DMExceptions.FilesMarkedKeep):
                    logger.debug(
                        "%s Failed action_validation.  Try next fileset" %
                        archiveme.result.resultsName)
                except DMExceptions.BaseInputLinked:
                    # want to allow Basecalling Input delete if all results are expired
                    relatedObjs = DMFileStat.objects.filter(
                        result__experiment=archiveme.result.experiment,
                        dmfileset__type=dmactions_types.BASE)
                    if relatedObjs.count() == relatedObjs.filter(
                            created__lt=threshdate).count():
                        archiveme.allow_delete = True
                        return archiveme
                    else:
                        logger.debug(
                            "%s Failed action_validation.  Try next fileset" %
                            archiveme.result.resultsName)
                except:
                    logger.error(traceback.format_exc())
            else:
                logger.debug("Skipped a preserved fileset: %s" %
                             archiveme.result.resultsName)

        logger.info("%d filestat objects are preserved." % dmfilestats.count())
        raise DMExceptions.NoDMFileStat("NONE FOUND")

    try:
        #logger.debug("manage_data lock for %s (%d)" % (dmfileset['type'], os.getpid()))
        #Create lock file to prevent more than one celery task for each process_type and partition
        lock_id = "%s_%s" % (hex(deviceid), slugify(dmfileset['type']))
        applock = TaskLock(lock_id)

        if not (applock.lock()):
            logger.info("Did not acquire lock file: %s" % lock_id)
            return

        logger.debug("Worker PID %d lock_id created %s" %
                     (os.getpid(), lock_id))

    except Exception as e:
        logger.exception(e)

    try:
        #---------------------------------------------------------------------------
        # Database object filtering
        #---------------------------------------------------------------------------
        actiondmfilestat = None
        user_comment = "Auto Action"
        # Order by incrementing pk.  This puts them in chronological order.
        # Select DMFileStat objects of category DMFileSet.type (1/4th of all objects)
        dmfilestats = DMFileStat.objects.filter(
            dmfileset__type=dmfileset['type']).order_by('created')
        tot_obj = dmfilestats.count()

        # Select objects not yet processed
        dmfilestats = dmfilestats.filter(action_state__in=['L', 'S', 'N', 'A'])
        tot_act = dmfilestats.count()
        logger.info("(%s)Total %s: %d Active: %d" %
                    (hex(deviceid), dmfileset['type'], tot_obj, tot_act))

        # Select objects that are old enough
        threshdate = datetime.now(
            pytz.UTC) - timedelta(days=dmfileset['auto_trigger_age'])
        dmfilestats = dmfilestats.filter(created__lt=threshdate)
        logger.info(
            "(%s)Total %s: %d Expired: %d" %
            (hex(deviceid), dmfileset['type'], tot_obj, dmfilestats.count()))

        # Select objects stored on the deviceid
        query = Q()
        for path in pathlist:
            if dmfileset['type'] == dmactions_types.SIG:
                query |= Q(result__experiment__expDir__startswith=path)
            # These categories have files in both directory paths.
            elif dmfileset['type'] in [
                    dmactions_types.INTR, dmactions_types.BASE
            ]:
                query |= Q(result__experiment__expDir__startswith=path)
                query |= Q(result__reportstorage__dirPath__startswith=path)
            else:
                query |= Q(result__reportstorage__dirPath__startswith=path)

        dmfilestats = dmfilestats.filter(query)
        logger.info("(%s)Total %s: %d On '%s' path: %d" %
                    (hex(deviceid), dmfileset['type'], tot_obj, path,
                     dmfilestats.count()))

        # Exclude objects marked 'Keep' upfront to optimize db access
        if dmfileset['type'] == dmactions_types.SIG:
            dmfilestats = dmfilestats.exclude(
                result__experiment__storage_options="KI")
        else:
            dmfilestats = dmfilestats.exclude(preserve_data=True)
        logger.info(
            "(%s)Total %s: %d Not Preserved: %d" %
            (hex(deviceid), dmfileset['type'], tot_obj, dmfilestats.count()))

        #---------------------------------------------------------------------------
        # Archive
        #---------------------------------------------------------------------------
        if dmfileset['auto_action'] == 'ARC':
            '''
            Rules:
            1) archive a fileset as soon as age threshold reached, regardless of disk threshold
            2) select oldest fileset
            3) select unprocessed fileset
            4) Archiving does not require 'S' -> 'N' -> 'A' progression before action
            5) Do not include filesets marked 'E' in auto-action - can get stuck on that fileset forever
            '''
            logger.info("(%s)Exceed %d days. Eligible to archive: %d" %
                        (hex(deviceid), dmfileset['auto_trigger_age'],
                         dmfilestats.count()))
            actiondmfilestat = None
            #Bail out if disabled
            if auto_action_enabled != True:
                logger.info("Data management auto-action is disabled.")
                applock.unlock()
                logger.debug("Worker PID %d lock_id destroyed %s" %
                             (os.getpid(), lock_id))
                return

            # Select first object stored on the deviceid
            try:
                actiondmfilestat = getfirstnotpreserved(
                    dmfilestats, dmactions.ARCHIVE, threshdate)
                logger.info(
                    "(%s)Picked: %s" %
                    (hex(deviceid), actiondmfilestat.result.resultsName))
            except DMExceptions.NoDMFileStat:
                logger.debug("No filesets to archive on this device: %s" %
                             hex(deviceid))
                applock.unlock()
                logger.debug("Worker PID %d lock_id destroyed %s" %
                             (os.getpid(), lock_id))
            except:
                logger.error(traceback.format_exc())
                applock.unlock()
                logger.debug("Worker PID %d lock_id destroyed %s" %
                             (os.getpid(), lock_id))
            else:
                archive_action('dm_agent', user_comment, actiondmfilestat,
                               lock_id)

        #---------------------------------------------------------------------------
        # Delete
        #---------------------------------------------------------------------------
        elif dmfileset['auto_action'] == 'DEL':
            '''
            Rules: If auto-acknowledge is True:
            promote to Acknowledged and delete when age and disk threshold reached.
            If auto-acknowledge is False:
                If fileset type is SIG:
                    'S' -> 'N' -> 'A' progression
                Else:
                    promote to 'A'
                delete an 'A' fileset
            '''
            logger.info("(%s)Exceed %d days. Eligible to delete: %d" %
                        (hex(deviceid), dmfileset['auto_trigger_age'],
                         dmfilestats.count()))
            if auto_acknowledge_enabled:
                if dmfileset['type'] == dmactions_types.SIG:
                    logger.debug(
                        "Sig Proc Input Files auto acknowledge enabled")
                '''
                Do not require 'S' -> 'N' -> 'A' progression before action; mark 'A' and process
                '''
                A_list = dmfilestats.filter(action_state='A')
                if A_list.count() > 0:
                    # there are filesets acknowledged and ready to be deleted.  Covers situation where user has
                    #already acknowledged but recently enabled auto-acknowledge as well.
                    #deleteme = A_list[0]
                    try:
                        actiondmfilestat = getfirstnotpreserved(
                            A_list, dmactions.DELETE, threshdate)
                        logger.info("(%s)Picked: %s" %
                                    (hex(deviceid),
                                     actiondmfilestat.result.resultsName))
                    except DMExceptions.NoDMFileStat:
                        logger.info(
                            "(%s) No filesets to delete on this device" %
                            hex(deviceid))
                        applock.unlock()
                        logger.debug("Worker PID %d lock_id destroyed %s" %
                                     (os.getpid(), lock_id))
                    except:
                        logger.error(traceback.format_exc())
                        applock.unlock()
                        logger.debug("Worker PID %d lock_id destroyed %s" %
                                     (os.getpid(), lock_id))

                if actiondmfilestat == None:
                    # Select oldest fileset regardless if its 'L','S','N','A'.  This covers situation where user
                    # recently enabled auto-acknowledge
                    try:
                        actiondmfilestat = getfirstnotpreserved(
                            dmfilestats, dmactions.DELETE, threshdate)
                        logger.info("(%s)Picked: %s" %
                                    (hex(deviceid),
                                     actiondmfilestat.result.resultsName))
                    except DMExceptions.NoDMFileStat:
                        logger.info(
                            "(%s) No filesets to delete on this device" %
                            hex(deviceid))
                        applock.unlock()
                        logger.debug("Worker PID %d lock_id destroyed %s" %
                                     (os.getpid(), lock_id))
                    except:
                        logger.error(traceback.format_exc())
                        applock.unlock()
                        logger.debug("Worker PID %d lock_id destroyed %s" %
                                     (os.getpid(), lock_id))
            else:
                if dmfileset['type'] == dmactions_types.SIG:
                    logger.debug(
                        "Sig Proc Input Files auto acknowledge disabled")
                    '''
                    Need to select # of filesets and mark 'S'
                    Need to find 'S' filesets and notify (set to 'N')
                    Set queue length to 5
                    sum(S,N,A) == queue length
                    '''
                    #Get email recipient
                    try:
                        recipient = User.objects.get(
                            username='******').email
                        recipient = recipient.replace(',', ' ').replace(
                            ';', ' ').split()
                    except:
                        recipient = None

                    # Need to select Experiments to process with appropriate dmfilestats action states.
                    exps = Experiment.objects.exclude(
                        storage_options='KI').exclude(
                            expDir='').order_by('date')
                    L_list = exps.filter(
                        results_set__dmfilestat__in=dmfilestats.filter(
                            action_state='L')).distinct()
                    S_list = exps.filter(
                        results_set__dmfilestat__in=dmfilestats.filter(
                            action_state='S')).distinct()
                    N_list = exps.filter(
                        results_set__dmfilestat__in=dmfilestats.filter(
                            action_state='N')).distinct()
                    A_list = exps.filter(
                        results_set__dmfilestat__in=dmfilestats.filter(
                            action_state='A')).distinct()

                    logger.info(
                        "Experiments (Keep=False) L:%d S:%d N:%d A:%d" %
                        (L_list.count(), S_list.count(), N_list.count(),
                         A_list.count()))
                    queue_length = 5
                    to_select = queue_length - (
                        A_list.count() + N_list.count() + S_list.count())
                    if to_select > 0:
                        # Mark to_select number of oldest Experiments, from 'L', to 'S'
                        promoted = dmfilestats.filter(
                            result__experiment__id__in=list(
                                L_list[:to_select].values_list('id',
                                                               flat=True)))
                        if auto_action_enabled:
                            promoted.update(action_state='S')
                            for dmfilestat in promoted:
                                EventLog.objects.add_entry(
                                    dmfilestat.result,
                                    "Signal Processing Input Selected for Deletion",
                                    username='******')

                    # Get updated list of Selected items
                    selected = dmfilestats.filter(action_state='S')

                    # Send Selected items to be notified
                    if selected.count() > 0 and auto_action_enabled:
                        logger.debug("notify recipient %s" % (recipient))
                        if notify([
                                dmfilestat.result.resultsName
                                for dmfilestat in selected
                        ], recipient):
                            selected.update(action_state='N')
                            for dmfilestat in selected:
                                EventLog.objects.add_entry(
                                    dmfilestat.result,
                                    "Notification for Deletion Sent",
                                    username='******')

                    try:
                        actiondmfilestat = getfirstnotpreserved(
                            dmfilestats.filter(action_state='A'),
                            dmactions.DELETE, threshdate)
                        logger.info("(%s)Picked: %s" %
                                    (hex(deviceid),
                                     actiondmfilestat.result.resultsName))
                    except DMExceptions.NoDMFileStat:
                        logger.info(
                            "(%s) No filesets to delete on this device" %
                            hex(deviceid))
                        applock.unlock()
                        logger.debug("Worker PID %d lock_id destroyed %s" %
                                     (os.getpid(), lock_id))
                    except:
                        logger.error(traceback.format_exc())
                        applock.unlock()
                        logger.debug("Worker PID %d lock_id destroyed %s" %
                                     (os.getpid(), lock_id))

                else:
                    try:
                        actiondmfilestat = getfirstnotpreserved(
                            dmfilestats, dmactions.DELETE, threshdate)
                        logger.info("(%s)Picked: %s" %
                                    (hex(deviceid),
                                     actiondmfilestat.result.resultsName))
                    except DMExceptions.NoDMFileStat:
                        logger.info(
                            "(%s) No filesets to delete on this device" %
                            hex(deviceid))
                        applock.unlock()
                        logger.debug("Worker PID %d lock_id destroyed %s" %
                                     (os.getpid(), lock_id))
                    except:
                        logger.error(traceback.format_exc())
                        applock.unlock()
                        logger.debug("Worker PID %d lock_id destroyed %s" %
                                     (os.getpid(), lock_id))

            #Bail out if disabled
            if auto_action_enabled != True:
                logger.info("Data management auto-action is disabled.")
                applock.unlock()
                logger.debug("Worker PID %d lock_id destroyed %s" %
                             (os.getpid(), lock_id))
                return

            if actiondmfilestat is not None:
                delete_action('dm_agent',
                              user_comment,
                              actiondmfilestat,
                              lock_id,
                              confirmed=getattr(actiondmfilestat,
                                                'allow_delete', False))

        else:
            logger.error("Unknown or unhandled action: %s" %
                         dmfileset['auto_action'])
            applock.unlock()

    except (DMExceptions.FilePermission, DMExceptions.InsufficientDiskSpace,
            DMExceptions.MediaNotSet, DMExceptions.MediaNotAvailable,
            DMExceptions.FilesInUse) as e:
        applock.unlock()
        message = Message.objects.filter(tags__contains=e.tag)
        if not message:
            Message.error(e.message, tags=e.tag)
            #TODO: TS-6525: This logentry will repeat for an Archive action every 30 seconds until the cause of the exception is fixed.
            #at least, while the message banner is raised, suppress additional Log Entries.
            EventLog.objects.add_entry(actiondmfilestat.result,
                                       "%s - %s" %
                                       (dmfileset['type'], e.message),
                                       username='******')
    except DMExceptions.SrcDirDoesNotExist as e:
        applock.unlock()
        msg = "Src Dir not found: %s. Setting action_state to Deleted" % e.message
        EventLog.objects.add_entry(actiondmfilestat.result,
                                   msg,
                                   username='******')
        actiondmfilestat.setactionstate('DD')
        logger.info(msg)
    except Exception as inst:
        applock.unlock()
        msg = ''
        if actiondmfilestat:
            msg = "Auto-action error on %s " % actiondmfilestat.result.resultsName
        msg += " Error: %s" % str(inst)
        logger.exception("%s - %s" % (dmfileset['type'], msg))

        if actiondmfilestat:
            EventLog.objects.add_entry(actiondmfilestat.result,
                                       "%s - %s" % (dmfileset['type'], msg),
                                       username='******')

    return