def project_msg_banner(user, project_msg, action): try: msg = "" thistag = "" logger.debug("Function: %s()" % sys._getframe().f_code.co_name) for pk, status_list in project_msg.iteritems(): url = urlresolvers.reverse("dm_log", args=(pk,)) report = Results.objects.get(id=pk) msg += "(%s) %s " % (report.resultsName, action.title()) for category, status in status_list.iteritems(): msg += " %s, " % str(category) msg += status grpstatus = status thistag = "%s_%s_%s" % (str(pk), action, slugify(category)) msg += " <a href='%s' data-toggle='modal' data-target='#modal_report_log'>View Report Log</a></br>" % (url) logger.debug("MESSAGE: %s" % msg) # If message already exists (ie, scheduled task) delete it. Message.objects.filter(tags=thistag).delete() if len(project_msg) > 1: thistag = "%s_%s_%s" % ("project", action, slugify(category)) if grpstatus == "scheduled": func = Message.info elif grpstatus == "success": func = Message.success else: func = Message.error except: func = Message.error logger.exception(traceback.format_exc()) return func(msg, route=user, tags=thistag)
def show_keepers(): for dmtype in FILESET_TYPES: # filename stem storename = "%s_%s_keepers" % (slugify(dmtype), gethostname()) # All SigProc Stat objects sigprocstats = models.DMFileStat.objects.filter(dmfileset__type=dmtype).order_by('created') print "All %s objects count: %d" % (dmtype, sigprocstats.count()) # Limit to objects marked keep sigprocstats = sigprocstats.filter(result__experiment__storage_options = 'KI') print "All %s objects Local marked Keep count: %d" % (dmtype, sigprocstats.count()) # Limit to objects with files on filesystem sigprocstats = sigprocstats.filter(action_state='L') print "All %s objects Local count: %d" % (dmtype, sigprocstats.count()) _write_csv(sigprocstats, storename) _write_excel(sigprocstats, storename)
def show_keepers(): for dmtype in FILESET_TYPES: # filename stem storename = "%s_%s_keepers" % (slugify(dmtype), gethostname()) # All SigProc Stat objects sigprocstats = models.DMFileStat.objects.filter( dmfileset__type=dmtype).order_by('created') print "All %s objects count: %d" % (dmtype, sigprocstats.count()) # Limit to objects marked keep sigprocstats = sigprocstats.filter( result__experiment__storage_options='KI') print "All %s objects Local marked Keep count: %d" % ( dmtype, sigprocstats.count()) # Limit to objects with files on filesystem sigprocstats = sigprocstats.filter(action_state='L') print "All %s objects Local count: %d" % (dmtype, sigprocstats.count()) _write_csv(sigprocstats, storename) _write_excel(sigprocstats, storename)
def fileserver_space_check(): '''For each file server, compare disk usage to backup threshold value. If disk usage exceeds threshold, launch celery task to delete/archive raw data directories. ''' logger = get_task_logger('data_management') # Get GlobalConfig object in order to access auto-acknowledge bit gc = GlobalConfig.get() auto_acknowledge = gc.auto_archive_ack auto_action_enabled = gc.auto_archive_enable # Get list of File Server objects file_servers = FileServer.objects.all().order_by('pk').values() # Get list of Report Storage objects report_storages = ReportStorage.objects.all().order_by('pk').values() # dict of fileset cateogries each with list of partition ids that can be acted upon. category_list = {} #------------------------------------------------- # DELETE action only happens if threshold reached #------------------------------------------------- for dmfileset in DMFileSet.objects.filter(version=settings.RELVERSION).filter(auto_action='DEL').values(): cat_name = slugify(dmfileset['type']) category_list[cat_name] = { 'dmfileset':dmfileset, 'devlist':[], 'partitions':[], } for partition in _partitions(file_servers,report_storages): if partition['diskusage'] >= dmfileset['auto_trigger_usage']: logger.info("%s %s %.2f%% exceeds %s %.0f%%" % ( hex(partition['devid']), partition['path'], partition['diskusage'], dmfileset['type'], dmfileset['auto_trigger_usage'])) category_list[cat_name]['devlist'].append(partition['devid']) category_list[cat_name]['partitions'].append(partition) else: logger.info("%s %s %.2f%% below %s %.0f%%" % ( hex(partition['devid']), partition['path'], partition['diskusage'], dmfileset['type'], dmfileset['auto_trigger_usage'])) # uniquify the deviceid list category_list[cat_name]['devlist'] = list(set(category_list[cat_name]['devlist'])) #------------------------------------------------------------------------------- #ARCHIVE action happens as soon as grace period has expired (no threshold check) #------------------------------------------------------------------------------- for dmfileset in DMFileSet.objects.filter(version=settings.RELVERSION).filter(auto_action='ARC').values(): cat_name = slugify(dmfileset['type']) category_list[cat_name] = { 'dmfileset':dmfileset, 'devlist':[], 'partitions':[], } for partition in _partitions(file_servers,report_storages): logger.debug("%s %s" %( partition['path'],hex(partition['devid']))) category_list[cat_name]['devlist'].append(partition['devid']) category_list[cat_name]['partitions'].append(partition) # uniquify the deviceid list category_list[cat_name]['devlist'] = list(set(category_list[cat_name]['devlist'])) #------------------------------------------------------------- # Action loop - for each category, launch action per deviceid #------------------------------------------------------------- if not auto_action_enabled: logger.info("Data management auto-action is disabled.") # update any DMFileStats that are in use by active analysis jobs try: update_files_in_use() except: logger.error('Unable to update active DMFileStats') logger.error(traceback.format_exc()) # Checks for manual export and archive requests. manage_manual_action.delay() for category,dict in category_list.iteritems(): for deviceid in dict['devlist']: pathlist = [item['path'] for item in dict['partitions'] if item['devid'] == deviceid] manage_data.delay(deviceid, dict['dmfileset'], pathlist, auto_acknowledge, auto_action_enabled) return
def manage_data(deviceid, dmfileset, pathlist, auto_acknowledge_enabled, auto_action_enabled): logger.debug("manage_data: %s %s %s" % (dmfileset['auto_action'], hex(deviceid),dmfileset['type'])) def getfirstnotpreserved(dmfilestats, action): '''QuerySet of DMFileStat objects. Returns first instance of an object with preserve_data set to False and passes the action_validation test''' logger.debug("Function: %s()" % sys._getframe().f_code.co_name) logger.debug("Looking at %d dmfilestat objects" % dmfilestats.count()) for archiveme in dmfilestats: if not archiveme.getpreserved(): try: dmactions.action_validation(archiveme,action) return archiveme except: logger.debug("%s Failed action_validation. Try next fileset" % archiveme.result.resultsName) else: logger.debug("Skipped a preserved fileset: %s" % archiveme.result.resultsName) logger.info("%d filestat objects are preserved." % dmfilestats.count()) raise DMExceptions.NoDMFileStat("NONE FOUND") try: #logger.debug("manage_data lock for %s (%d)" % (dmfileset['type'], os.getpid())) #Create lock file to prevent more than one celery task for each process_type and partition uniqid = "%s_%s" % (hex(deviceid),slugify(dmfileset['type'])) lockfile = '/var/run/celery/%s.lock' % (uniqid) applock = ApplicationLock(lockfile) if not(applock.lock()): logger.info("failed to acquire lock file: %s" % uniqid) return logger.info("Worker PID %d lockfile created %s" % (os.getpid(),uniqid)) except Exception as e: logger.exception(e) try: #--------------------------------------------------------------------------- # Database object filtering #--------------------------------------------------------------------------- actiondmfilestat = None user_comment = "Auto Action" # Order by incrementing pk. This puts them in chronological order. # Select DMFileStat objects of category DMFileSet.type (1/4th of all objects) dmfilestats = DMFileStat.objects.filter(dmfileset__type=dmfileset['type']).order_by('created') tot_obj = dmfilestats.count() # Select objects not yet processed dmfilestats = dmfilestats.filter(action_state__in=['L','S','N','A']) tot_act = dmfilestats.count() logger.info("(%s)Total %s: %d Active: %d" %(hex(deviceid),dmfileset['type'],tot_obj,tot_act)) # Select objects that are old enough threshdate = datetime.now(pytz.UTC) - timedelta(days=dmfileset['auto_trigger_age']) dmfilestats = dmfilestats.filter(created__lt=threshdate) # Select objects stored on the deviceid query = Q() for path in pathlist: if dmfileset['type'] == dmactions_types.SIG: dmfilestats = dmfilestats.filter(result__experiment__expDir__startswith=path) else: dmfilestats = dmfilestats.filter(result__reportstorage__dirPath__startswith=path) # query |= Q(result__experiment__expDir__startswith=path) | Q(result__reportstorage__dirPath__startswith=path) #dmfilestats = dmfilestats.filter(query) #--------------------------------------------------------------------------- # Archive #--------------------------------------------------------------------------- if dmfileset['auto_action'] == 'ARC': ''' Rules: 1) archive a fileset as soon as age threshold reached, regardless of disk threshold 2) select oldest fileset 3) select unprocessed fileset 4) Archiving does not require 'S' -> 'N' -> 'A' progression before action 5) Do not include filesets marked 'E' in auto-action - can get stuck on that fileset forever ''' logger.info("(%s)Exceed %d days. Eligible to archive: %d" %(hex(deviceid),dmfileset['auto_trigger_age'],dmfilestats.count())) actiondmfilestat = None #Bail out if disabled if auto_action_enabled != True: logger.info("Data management auto-action is disabled.") return # Select first object stored on the deviceid try: actiondmfilestat = getfirstnotpreserved(dmfilestats, dmactions.ARCHIVE) logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName)) except DMExceptions.NoDMFileStat: logger.debug("No filesets to archive on this device: %s" % hex(deviceid)) except: logger.error(traceback.format_exc()) else: archive_action('dm_agent', user_comment, actiondmfilestat) #--------------------------------------------------------------------------- # Delete #--------------------------------------------------------------------------- elif dmfileset['auto_action'] == 'DEL': ''' Rules: If auto-acknowledge is True: promote to Acknowledged and delete when age and disk threshold reached. If auto-acknowledge is False: If fileset type is SIG: 'S' -> 'N' -> 'A' progression Else: promote to 'A' delete an 'A' fileset ''' logger.info("(%s)Exceed %d days. Eligible to delete: %d" %(hex(deviceid),dmfileset['auto_trigger_age'],dmfilestats.count())) if auto_acknowledge_enabled: if dmfileset['type'] == dmactions_types.SIG: logger.debug("Sig Proc Input Files auto acknowledge enabled") ''' Do not require 'S' -> 'N' -> 'A' progression before action; mark 'A' and process ''' A_list = dmfilestats.filter(action_state='A') if A_list.count() > 0: # there are filesets acknowledged and ready to be deleted. Covers situation where user has #already acknowledged but recently enabled auto-acknowledge as well. #deleteme = A_list[0] try: actiondmfilestat = getfirstnotpreserved(A_list, dmactions.DELETE) logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName)) except DMExceptions.NoDMFileStat: logger.info("(%s) No filesets to delete on this device" % hex(deviceid)) except: logger.error(traceback.format_exc()) if actiondmfilestat == None: # Select oldest fileset regardless if its 'L','S','N','A'. This covers situation where user # recently enabled auto-acknowledge try: actiondmfilestat = getfirstnotpreserved(dmfilestats, dmactions.DELETE) logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName)) except DMExceptions.NoDMFileStat: logger.info("(%s) No filesets to delete on this device" % hex(deviceid)) except: logger.error(traceback.format_exc()) else: if dmfileset['type'] == dmactions_types.SIG: logger.debug("Sig Proc Input Files auto acknowledge disabled") ''' Need to select # of filesets and mark 'S' Need to find 'S' filesets and notify (set to 'N') Set queue length to 5 sum(S,N,A) == queue length ''' #Get email recipient try: recipient = User.objects.get(username='******').email recipient = recipient.replace(',',' ').replace(';',' ').split() except: recipient = None # Need to select Experiments to process with appropriate dmfilestats action states. exps = Experiment.objects.exclude(storage_options='KI').exclude(expDir='').order_by('date') L_list = exps.filter(results_set__dmfilestat__in = dmfilestats.filter(action_state='L')).distinct() S_list = exps.filter(results_set__dmfilestat__in = dmfilestats.filter(action_state='S')).distinct() N_list = exps.filter(results_set__dmfilestat__in = dmfilestats.filter(action_state='N')).distinct() A_list = exps.filter(results_set__dmfilestat__in = dmfilestats.filter(action_state='A')).distinct() logger.info("Experiments (Keep=False) L:%d S:%d N:%d A:%d" % (L_list.count(),S_list.count(),N_list.count(),A_list.count())) queue_length = 5 to_select = queue_length - (A_list.count() + N_list.count() + S_list.count()) if to_select > 0: # Mark to_select number of oldest Experiments, from 'L', to 'S' promoted = dmfilestats.filter(result__experiment__id__in=list(L_list[:to_select].values_list('id',flat=True))) if auto_action_enabled: promoted.update(action_state = 'S') for dmfilestat in promoted: EventLog.objects.add_entry(dmfilestat.result, "Signal Processing Input Selected for Deletion", username='******') # Get updated list of Selected items selected = dmfilestats.filter(action_state='S') # Send Selected items to be notified if selected.count() > 0 and auto_action_enabled: logger.debug("notify recipient %s" % (recipient)) if notify([dmfilestat.result.resultsName for dmfilestat in selected],recipient): selected.update(action_state = 'N') for dmfilestat in selected: EventLog.objects.add_entry(dmfilestat.result, "Notification for Deletion Sent", username='******') try: actiondmfilestat = getfirstnotpreserved(dmfilestats.filter(action_state='A'), dmactions.DELETE) logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName)) except DMExceptions.NoDMFileStat: logger.info("(%s) No filesets to delete on this device" % hex(deviceid)) except: logger.error(traceback.format_exc()) else: try: actiondmfilestat = getfirstnotpreserved(dmfilestats, dmactions.DELETE) logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName)) except DMExceptions.NoDMFileStat: logger.info("(%s) No filesets to delete on this device" % hex(deviceid)) except: logger.error(traceback.format_exc()) #Bail out if disabled if auto_action_enabled != True: logger.info("Data management auto-action is disabled.") return if actiondmfilestat is not None: delete_action('dm_agent', user_comment, actiondmfilestat) else: logger.error("Unknown or unhandled action: %s" % dmfileset['auto_action']) except (DMExceptions.FilePermission, DMExceptions.InsufficientDiskSpace, DMExceptions.MediaNotSet, DMExceptions.MediaNotAvailable, DMExceptions.FilesInUse) as e: message = Message.objects.filter(tags__contains=e.tag) if not message: Message.error(e.message,tags=e.tag) #TODO: TS-6525: This logentry will repeat for an Archive action every 30 seconds until the cause of the exception is fixed. #at least, while the message banner is raised, suppress additional Log Entries. EventLog.objects.add_entry(actiondmfilestat.result,"%s - %s" % (dmfileset['type'], e.message),username='******') except DMExceptions.SrcDirDoesNotExist as e: if actiondmfilestat.dmfileset.type == dmactions_types.SIG: msg = "Src Dir not found: %s. Setting action_state to Deleted" % e.message EventLog.objects.add_entry(actiondmfilestat.result,msg,username='******') actiondmfilestat.setactionstate('DD') logger.info(msg) except Exception as inst: msg = '' if actiondmfilestat: msg = "Auto-action error on %s " % actiondmfilestat.result.resultsName msg += " Error: %s" % str(inst) logger.exception("%s - %s" % (dmfileset['type'], msg)) if actiondmfilestat: EventLog.objects.add_entry(actiondmfilestat.result,"%s - %s" % (dmfileset['type'], msg),username='******') finally: applock.unlock() #logger.debug("(%05d,%05d) Release lock file" % (os.getppid(),os.getpid())) return
def fileserver_space_check(): '''For each file server, compare disk usage to backup threshold value. If disk usage exceeds threshold, launch celery task to delete/archive raw data directories. ''' # Get GlobalConfig object in order to access auto-acknowledge bit gc = GlobalConfig.get() auto_acknowledge = gc.auto_archive_ack auto_action_enabled = gc.auto_archive_enable # Get list of File Server objects file_servers = FileServer.objects.all().order_by('pk').values() # Get list of Report Storage objects report_storages = ReportStorage.objects.all().order_by('pk').values() # dict of fileset cateogries each with list of partition ids that can be acted upon. category_list = {} #------------------------------------------------- # DELETE action only happens if threshold reached #------------------------------------------------- for dmfileset in DMFileSet.objects.filter( version=settings.RELVERSION).filter(auto_action='DEL').values(): cat_name = slugify(dmfileset['type']) category_list[cat_name] = { 'dmfileset': dmfileset, 'devlist': [], 'partitions': [], } for partition in _partitions(file_servers, report_storages): if partition['diskusage'] >= dmfileset['auto_trigger_usage']: logger.info("%s %s %.2f%% exceeds %s %.0f%%" % (hex(partition['devid']), partition['path'], partition['diskusage'], dmfileset['type'], dmfileset['auto_trigger_usage'])) category_list[cat_name]['devlist'].append(partition['devid']) category_list[cat_name]['partitions'].append(partition) else: logger.info("%s %s %.2f%% below %s %.0f%%" % (hex(partition['devid']), partition['path'], partition['diskusage'], dmfileset['type'], dmfileset['auto_trigger_usage'])) # uniquify the deviceid list category_list[cat_name]['devlist'] = list( set(category_list[cat_name]['devlist'])) #------------------------------------------------------------------------------- #ARCHIVE action happens as soon as grace period has expired (no threshold check) #------------------------------------------------------------------------------- for dmfileset in DMFileSet.objects.filter( version=settings.RELVERSION).filter(auto_action='ARC').values(): cat_name = slugify(dmfileset['type']) category_list[cat_name] = { 'dmfileset': dmfileset, 'devlist': [], 'partitions': [], } for partition in _partitions(file_servers, report_storages): logger.debug("%s %s" % (partition['path'], hex(partition['devid']))) category_list[cat_name]['devlist'].append(partition['devid']) category_list[cat_name]['partitions'].append(partition) # uniquify the deviceid list category_list[cat_name]['devlist'] = list( set(category_list[cat_name]['devlist'])) #------------------------------------------------------------- # Action loop - for each category, launch action per deviceid #------------------------------------------------------------- if not auto_action_enabled: logger.info("Data management auto-action is disabled.") # update any DMFileStats that are in use by active analysis jobs try: update_files_in_use() except: logger.error('Unable to update active DMFileStats') logger.error(traceback.format_exc()) # Checks for manual export and archive requests. manage_manual_action.delay() for category, dict in category_list.iteritems(): for deviceid in dict['devlist']: pathlist = [ item['path'] for item in dict['partitions'] if item['devid'] == deviceid ] async_task_result = manage_data.delay(deviceid, dict['dmfileset'], pathlist, auto_acknowledge, auto_action_enabled) return
def manage_data(deviceid, dmfileset, pathlist, auto_acknowledge_enabled, auto_action_enabled): logger.debug("manage_data: %s %s %s" % (dmfileset['auto_action'], hex(deviceid), dmfileset['type'])) def getfirstnotpreserved(dmfilestats, action, threshdate): '''QuerySet of DMFileStat objects. Returns first instance of an object with preserve_data set to False and passes the action_validation test''' logger.debug("Function: %s()" % sys._getframe().f_code.co_name) logger.debug("Looking at %d dmfilestat objects" % dmfilestats.count()) for archiveme in dmfilestats: if not archiveme.getpreserved(): try: dmactions.action_validation(archiveme, action) return archiveme except (DMExceptions.FilesInUse, DMExceptions.FilesMarkedKeep): logger.debug( "%s Failed action_validation. Try next fileset" % archiveme.result.resultsName) except DMExceptions.BaseInputLinked: # want to allow Basecalling Input delete if all results are expired relatedObjs = DMFileStat.objects.filter( result__experiment=archiveme.result.experiment, dmfileset__type=dmactions_types.BASE) if relatedObjs.count() == relatedObjs.filter( created__lt=threshdate).count(): archiveme.allow_delete = True return archiveme else: logger.debug( "%s Failed action_validation. Try next fileset" % archiveme.result.resultsName) except: logger.error(traceback.format_exc()) else: logger.debug("Skipped a preserved fileset: %s" % archiveme.result.resultsName) logger.info("%d filestat objects are preserved." % dmfilestats.count()) raise DMExceptions.NoDMFileStat("NONE FOUND") try: #logger.debug("manage_data lock for %s (%d)" % (dmfileset['type'], os.getpid())) #Create lock file to prevent more than one celery task for each process_type and partition lock_id = "%s_%s" % (hex(deviceid), slugify(dmfileset['type'])) applock = TaskLock(lock_id) if not (applock.lock()): logger.info("Did not acquire lock file: %s" % lock_id) return logger.debug("Worker PID %d lock_id created %s" % (os.getpid(), lock_id)) except Exception as e: logger.exception(e) try: #--------------------------------------------------------------------------- # Database object filtering #--------------------------------------------------------------------------- actiondmfilestat = None user_comment = "Auto Action" # Order by incrementing pk. This puts them in chronological order. # Select DMFileStat objects of category DMFileSet.type (1/4th of all objects) dmfilestats = DMFileStat.objects.filter( dmfileset__type=dmfileset['type']).order_by('created') tot_obj = dmfilestats.count() # Select objects not yet processed dmfilestats = dmfilestats.filter(action_state__in=['L', 'S', 'N', 'A']) tot_act = dmfilestats.count() logger.info("(%s)Total %s: %d Active: %d" % (hex(deviceid), dmfileset['type'], tot_obj, tot_act)) # Select objects that are old enough threshdate = datetime.now( pytz.UTC) - timedelta(days=dmfileset['auto_trigger_age']) dmfilestats = dmfilestats.filter(created__lt=threshdate) logger.info( "(%s)Total %s: %d Expired: %d" % (hex(deviceid), dmfileset['type'], tot_obj, dmfilestats.count())) # Select objects stored on the deviceid query = Q() for path in pathlist: if dmfileset['type'] == dmactions_types.SIG: query |= Q(result__experiment__expDir__startswith=path) # These categories have files in both directory paths. elif dmfileset['type'] in [ dmactions_types.INTR, dmactions_types.BASE ]: query |= Q(result__experiment__expDir__startswith=path) query |= Q(result__reportstorage__dirPath__startswith=path) else: query |= Q(result__reportstorage__dirPath__startswith=path) dmfilestats = dmfilestats.filter(query) logger.info("(%s)Total %s: %d On '%s' path: %d" % (hex(deviceid), dmfileset['type'], tot_obj, path, dmfilestats.count())) # Exclude objects marked 'Keep' upfront to optimize db access if dmfileset['type'] == dmactions_types.SIG: dmfilestats = dmfilestats.exclude( result__experiment__storage_options="KI") else: dmfilestats = dmfilestats.exclude(preserve_data=True) logger.info( "(%s)Total %s: %d Not Preserved: %d" % (hex(deviceid), dmfileset['type'], tot_obj, dmfilestats.count())) #--------------------------------------------------------------------------- # Archive #--------------------------------------------------------------------------- if dmfileset['auto_action'] == 'ARC': ''' Rules: 1) archive a fileset as soon as age threshold reached, regardless of disk threshold 2) select oldest fileset 3) select unprocessed fileset 4) Archiving does not require 'S' -> 'N' -> 'A' progression before action 5) Do not include filesets marked 'E' in auto-action - can get stuck on that fileset forever ''' logger.info("(%s)Exceed %d days. Eligible to archive: %d" % (hex(deviceid), dmfileset['auto_trigger_age'], dmfilestats.count())) actiondmfilestat = None #Bail out if disabled if auto_action_enabled != True: logger.info("Data management auto-action is disabled.") applock.unlock() logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id)) return # Select first object stored on the deviceid try: actiondmfilestat = getfirstnotpreserved( dmfilestats, dmactions.ARCHIVE, threshdate) logger.info( "(%s)Picked: %s" % (hex(deviceid), actiondmfilestat.result.resultsName)) except DMExceptions.NoDMFileStat: logger.debug("No filesets to archive on this device: %s" % hex(deviceid)) applock.unlock() logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id)) except: logger.error(traceback.format_exc()) applock.unlock() logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id)) else: archive_action('dm_agent', user_comment, actiondmfilestat, lock_id) #--------------------------------------------------------------------------- # Delete #--------------------------------------------------------------------------- elif dmfileset['auto_action'] == 'DEL': ''' Rules: If auto-acknowledge is True: promote to Acknowledged and delete when age and disk threshold reached. If auto-acknowledge is False: If fileset type is SIG: 'S' -> 'N' -> 'A' progression Else: promote to 'A' delete an 'A' fileset ''' logger.info("(%s)Exceed %d days. Eligible to delete: %d" % (hex(deviceid), dmfileset['auto_trigger_age'], dmfilestats.count())) if auto_acknowledge_enabled: if dmfileset['type'] == dmactions_types.SIG: logger.debug( "Sig Proc Input Files auto acknowledge enabled") ''' Do not require 'S' -> 'N' -> 'A' progression before action; mark 'A' and process ''' A_list = dmfilestats.filter(action_state='A') if A_list.count() > 0: # there are filesets acknowledged and ready to be deleted. Covers situation where user has #already acknowledged but recently enabled auto-acknowledge as well. #deleteme = A_list[0] try: actiondmfilestat = getfirstnotpreserved( A_list, dmactions.DELETE, threshdate) logger.info("(%s)Picked: %s" % (hex(deviceid), actiondmfilestat.result.resultsName)) except DMExceptions.NoDMFileStat: logger.info( "(%s) No filesets to delete on this device" % hex(deviceid)) applock.unlock() logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id)) except: logger.error(traceback.format_exc()) applock.unlock() logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id)) if actiondmfilestat == None: # Select oldest fileset regardless if its 'L','S','N','A'. This covers situation where user # recently enabled auto-acknowledge try: actiondmfilestat = getfirstnotpreserved( dmfilestats, dmactions.DELETE, threshdate) logger.info("(%s)Picked: %s" % (hex(deviceid), actiondmfilestat.result.resultsName)) except DMExceptions.NoDMFileStat: logger.info( "(%s) No filesets to delete on this device" % hex(deviceid)) applock.unlock() logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id)) except: logger.error(traceback.format_exc()) applock.unlock() logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id)) else: if dmfileset['type'] == dmactions_types.SIG: logger.debug( "Sig Proc Input Files auto acknowledge disabled") ''' Need to select # of filesets and mark 'S' Need to find 'S' filesets and notify (set to 'N') Set queue length to 5 sum(S,N,A) == queue length ''' #Get email recipient try: recipient = User.objects.get( username='******').email recipient = recipient.replace(',', ' ').replace( ';', ' ').split() except: recipient = None # Need to select Experiments to process with appropriate dmfilestats action states. exps = Experiment.objects.exclude( storage_options='KI').exclude( expDir='').order_by('date') L_list = exps.filter( results_set__dmfilestat__in=dmfilestats.filter( action_state='L')).distinct() S_list = exps.filter( results_set__dmfilestat__in=dmfilestats.filter( action_state='S')).distinct() N_list = exps.filter( results_set__dmfilestat__in=dmfilestats.filter( action_state='N')).distinct() A_list = exps.filter( results_set__dmfilestat__in=dmfilestats.filter( action_state='A')).distinct() logger.info( "Experiments (Keep=False) L:%d S:%d N:%d A:%d" % (L_list.count(), S_list.count(), N_list.count(), A_list.count())) queue_length = 5 to_select = queue_length - ( A_list.count() + N_list.count() + S_list.count()) if to_select > 0: # Mark to_select number of oldest Experiments, from 'L', to 'S' promoted = dmfilestats.filter( result__experiment__id__in=list( L_list[:to_select].values_list('id', flat=True))) if auto_action_enabled: promoted.update(action_state='S') for dmfilestat in promoted: EventLog.objects.add_entry( dmfilestat.result, "Signal Processing Input Selected for Deletion", username='******') # Get updated list of Selected items selected = dmfilestats.filter(action_state='S') # Send Selected items to be notified if selected.count() > 0 and auto_action_enabled: logger.debug("notify recipient %s" % (recipient)) if notify([ dmfilestat.result.resultsName for dmfilestat in selected ], recipient): selected.update(action_state='N') for dmfilestat in selected: EventLog.objects.add_entry( dmfilestat.result, "Notification for Deletion Sent", username='******') try: actiondmfilestat = getfirstnotpreserved( dmfilestats.filter(action_state='A'), dmactions.DELETE, threshdate) logger.info("(%s)Picked: %s" % (hex(deviceid), actiondmfilestat.result.resultsName)) except DMExceptions.NoDMFileStat: logger.info( "(%s) No filesets to delete on this device" % hex(deviceid)) applock.unlock() logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id)) except: logger.error(traceback.format_exc()) applock.unlock() logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id)) else: try: actiondmfilestat = getfirstnotpreserved( dmfilestats, dmactions.DELETE, threshdate) logger.info("(%s)Picked: %s" % (hex(deviceid), actiondmfilestat.result.resultsName)) except DMExceptions.NoDMFileStat: logger.info( "(%s) No filesets to delete on this device" % hex(deviceid)) applock.unlock() logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id)) except: logger.error(traceback.format_exc()) applock.unlock() logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id)) #Bail out if disabled if auto_action_enabled != True: logger.info("Data management auto-action is disabled.") applock.unlock() logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id)) return if actiondmfilestat is not None: delete_action('dm_agent', user_comment, actiondmfilestat, lock_id, confirmed=getattr(actiondmfilestat, 'allow_delete', False)) else: logger.error("Unknown or unhandled action: %s" % dmfileset['auto_action']) applock.unlock() except (DMExceptions.FilePermission, DMExceptions.InsufficientDiskSpace, DMExceptions.MediaNotSet, DMExceptions.MediaNotAvailable, DMExceptions.FilesInUse) as e: applock.unlock() message = Message.objects.filter(tags__contains=e.tag) if not message: Message.error(e.message, tags=e.tag) #TODO: TS-6525: This logentry will repeat for an Archive action every 30 seconds until the cause of the exception is fixed. #at least, while the message banner is raised, suppress additional Log Entries. EventLog.objects.add_entry(actiondmfilestat.result, "%s - %s" % (dmfileset['type'], e.message), username='******') except DMExceptions.SrcDirDoesNotExist as e: applock.unlock() msg = "Src Dir not found: %s. Setting action_state to Deleted" % e.message EventLog.objects.add_entry(actiondmfilestat.result, msg, username='******') actiondmfilestat.setactionstate('DD') logger.info(msg) except Exception as inst: applock.unlock() msg = '' if actiondmfilestat: msg = "Auto-action error on %s " % actiondmfilestat.result.resultsName msg += " Error: %s" % str(inst) logger.exception("%s - %s" % (dmfileset['type'], msg)) if actiondmfilestat: EventLog.objects.add_entry(actiondmfilestat.result, "%s - %s" % (dmfileset['type'], msg), username='******') return