def project_msg_banner(user, project_msg, action):
try:
msg = ""
thistag = ""
logger.debug("Function: %s()" % sys._getframe().f_code.co_name)
for pk, status_list in project_msg.iteritems():
url = urlresolvers.reverse("dm_log", args=(pk,))
report = Results.objects.get(id=pk)
msg += "(%s) %s " % (report.resultsName, action.title())
for category, status in status_list.iteritems():
msg += " %s, " % str(category)
msg += status
grpstatus = status
thistag = "%s_%s_%s" % (str(pk), action, slugify(category))
msg += " <a href='%s' data-toggle='modal' data-target='#modal_report_log'>View Report Log</a></br>" % (url)
logger.debug("MESSAGE: %s" % msg)
# If message already exists (ie, scheduled task) delete it.
Message.objects.filter(tags=thistag).delete()
if len(project_msg) > 1:
thistag = "%s_%s_%s" % ("project", action, slugify(category))
if grpstatus == "scheduled":
func = Message.info
elif grpstatus == "success":
func = Message.success
else:
func = Message.error
except:
func = Message.error
logger.exception(traceback.format_exc())
return func(msg, route=user, tags=thistag)
def show_keepers():
for dmtype in FILESET_TYPES:
# filename stem
storename = "%s_%s_keepers" % (slugify(dmtype), gethostname())
# All SigProc Stat objects
sigprocstats = models.DMFileStat.objects.filter(dmfileset__type=dmtype).order_by('created')
print "All %s objects count: %d" % (dmtype, sigprocstats.count())
# Limit to objects marked keep
sigprocstats = sigprocstats.filter(result__experiment__storage_options = 'KI')
print "All %s objects Local marked Keep count: %d" % (dmtype, sigprocstats.count())
# Limit to objects with files on filesystem
sigprocstats = sigprocstats.filter(action_state='L')
print "All %s objects Local count: %d" % (dmtype, sigprocstats.count())
_write_csv(sigprocstats, storename)
_write_excel(sigprocstats, storename)
def show_keepers():
for dmtype in FILESET_TYPES:
# filename stem
storename = "%s_%s_keepers" % (slugify(dmtype), gethostname())
# All SigProc Stat objects
sigprocstats = models.DMFileStat.objects.filter(
dmfileset__type=dmtype).order_by('created')
print "All %s objects count: %d" % (dmtype, sigprocstats.count())
# Limit to objects marked keep
sigprocstats = sigprocstats.filter(
result__experiment__storage_options='KI')
print "All %s objects Local marked Keep count: %d" % (
dmtype, sigprocstats.count())
# Limit to objects with files on filesystem
sigprocstats = sigprocstats.filter(action_state='L')
print "All %s objects Local count: %d" % (dmtype, sigprocstats.count())
_write_csv(sigprocstats, storename)
_write_excel(sigprocstats, storename)
def fileserver_space_check():
'''For each file server, compare disk usage to backup threshold value.
If disk usage exceeds threshold, launch celery task to delete/archive
raw data directories.
'''
logger = get_task_logger('data_management')
# Get GlobalConfig object in order to access auto-acknowledge bit
gc = GlobalConfig.get()
auto_acknowledge = gc.auto_archive_ack
auto_action_enabled = gc.auto_archive_enable
# Get list of File Server objects
file_servers = FileServer.objects.all().order_by('pk').values()
# Get list of Report Storage objects
report_storages = ReportStorage.objects.all().order_by('pk').values()
# dict of fileset cateogries each with list of partition ids that can be acted upon.
category_list = {}
#-------------------------------------------------
# DELETE action only happens if threshold reached
#-------------------------------------------------
for dmfileset in DMFileSet.objects.filter(version=settings.RELVERSION).filter(auto_action='DEL').values():
cat_name = slugify(dmfileset['type'])
category_list[cat_name] = {
'dmfileset':dmfileset,
'devlist':[],
'partitions':[],
}
for partition in _partitions(file_servers,report_storages):
if partition['diskusage'] >= dmfileset['auto_trigger_usage']:
logger.info("%s %s %.2f%% exceeds %s %.0f%%" % (
hex(partition['devid']),
partition['path'],
partition['diskusage'],
dmfileset['type'],
dmfileset['auto_trigger_usage']))
category_list[cat_name]['devlist'].append(partition['devid'])
category_list[cat_name]['partitions'].append(partition)
else:
logger.info("%s %s %.2f%% below %s %.0f%%" % (
hex(partition['devid']),
partition['path'],
partition['diskusage'],
dmfileset['type'],
dmfileset['auto_trigger_usage']))
# uniquify the deviceid list
category_list[cat_name]['devlist'] = list(set(category_list[cat_name]['devlist']))
#-------------------------------------------------------------------------------
#ARCHIVE action happens as soon as grace period has expired (no threshold check)
#-------------------------------------------------------------------------------
for dmfileset in DMFileSet.objects.filter(version=settings.RELVERSION).filter(auto_action='ARC').values():
cat_name = slugify(dmfileset['type'])
category_list[cat_name] = {
'dmfileset':dmfileset,
'devlist':[],
'partitions':[],
}
for partition in _partitions(file_servers,report_storages):
logger.debug("%s %s" %( partition['path'],hex(partition['devid'])))
category_list[cat_name]['devlist'].append(partition['devid'])
category_list[cat_name]['partitions'].append(partition)
# uniquify the deviceid list
category_list[cat_name]['devlist'] = list(set(category_list[cat_name]['devlist']))
#-------------------------------------------------------------
# Action loop - for each category, launch action per deviceid
#-------------------------------------------------------------
if not auto_action_enabled:
logger.info("Data management auto-action is disabled.")
# update any DMFileStats that are in use by active analysis jobs
try:
update_files_in_use()
except:
logger.error('Unable to update active DMFileStats')
logger.error(traceback.format_exc())
# Checks for manual export and archive requests.
manage_manual_action.delay()
for category,dict in category_list.iteritems():
for deviceid in dict['devlist']:
pathlist = [item['path'] for item in dict['partitions'] if item['devid'] == deviceid]
manage_data.delay(deviceid, dict['dmfileset'], pathlist, auto_acknowledge, auto_action_enabled)
return
def manage_data(deviceid, dmfileset, pathlist, auto_acknowledge_enabled, auto_action_enabled):
logger.debug("manage_data: %s %s %s" % (dmfileset['auto_action'], hex(deviceid),dmfileset['type']))
def getfirstnotpreserved(dmfilestats, action):
'''QuerySet of DMFileStat objects. Returns first instance of an object
with preserve_data set to False and passes the action_validation test'''
logger.debug("Function: %s()" % sys._getframe().f_code.co_name)
logger.debug("Looking at %d dmfilestat objects" % dmfilestats.count())
for archiveme in dmfilestats:
if not archiveme.getpreserved():
try:
dmactions.action_validation(archiveme,action)
return archiveme
except:
logger.debug("%s Failed action_validation. Try next fileset" % archiveme.result.resultsName)
else:
logger.debug("Skipped a preserved fileset: %s" % archiveme.result.resultsName)
logger.info("%d filestat objects are preserved." % dmfilestats.count())
raise DMExceptions.NoDMFileStat("NONE FOUND")
try:
#logger.debug("manage_data lock for %s (%d)" % (dmfileset['type'], os.getpid()))
#Create lock file to prevent more than one celery task for each process_type and partition
uniqid = "%s_%s" % (hex(deviceid),slugify(dmfileset['type']))
lockfile = '/var/run/celery/%s.lock' % (uniqid)
applock = ApplicationLock(lockfile)
if not(applock.lock()):
logger.info("failed to acquire lock file: %s" % uniqid)
return
logger.info("Worker PID %d lockfile created %s" % (os.getpid(),uniqid))
except Exception as e:
logger.exception(e)
try:
#---------------------------------------------------------------------------
# Database object filtering
#---------------------------------------------------------------------------
actiondmfilestat = None
user_comment = "Auto Action"
# Order by incrementing pk. This puts them in chronological order.
# Select DMFileStat objects of category DMFileSet.type (1/4th of all objects)
dmfilestats = DMFileStat.objects.filter(dmfileset__type=dmfileset['type']).order_by('created')
tot_obj = dmfilestats.count()
# Select objects not yet processed
dmfilestats = dmfilestats.filter(action_state__in=['L','S','N','A'])
tot_act = dmfilestats.count()
logger.info("(%s)Total %s: %d Active: %d" %(hex(deviceid),dmfileset['type'],tot_obj,tot_act))
# Select objects that are old enough
threshdate = datetime.now(pytz.UTC) - timedelta(days=dmfileset['auto_trigger_age'])
dmfilestats = dmfilestats.filter(created__lt=threshdate)
# Select objects stored on the deviceid
query = Q()
for path in pathlist:
if dmfileset['type'] == dmactions_types.SIG:
dmfilestats = dmfilestats.filter(result__experiment__expDir__startswith=path)
else:
dmfilestats = dmfilestats.filter(result__reportstorage__dirPath__startswith=path)
# query |= Q(result__experiment__expDir__startswith=path) | Q(result__reportstorage__dirPath__startswith=path)
#dmfilestats = dmfilestats.filter(query)
#---------------------------------------------------------------------------
# Archive
#---------------------------------------------------------------------------
if dmfileset['auto_action'] == 'ARC':
'''
Rules:
1) archive a fileset as soon as age threshold reached, regardless of disk threshold
2) select oldest fileset
3) select unprocessed fileset
4) Archiving does not require 'S' -> 'N' -> 'A' progression before action
5) Do not include filesets marked 'E' in auto-action - can get stuck on that fileset forever
'''
logger.info("(%s)Exceed %d days. Eligible to archive: %d" %(hex(deviceid),dmfileset['auto_trigger_age'],dmfilestats.count()))
actiondmfilestat = None
#Bail out if disabled
if auto_action_enabled != True:
logger.info("Data management auto-action is disabled.")
return
# Select first object stored on the deviceid
try:
actiondmfilestat = getfirstnotpreserved(dmfilestats, dmactions.ARCHIVE)
logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName))
except DMExceptions.NoDMFileStat:
logger.debug("No filesets to archive on this device: %s" % hex(deviceid))
except:
logger.error(traceback.format_exc())
else:
archive_action('dm_agent', user_comment, actiondmfilestat)
#---------------------------------------------------------------------------
# Delete
#---------------------------------------------------------------------------
elif dmfileset['auto_action'] == 'DEL':
'''
Rules: If auto-acknowledge is True:
promote to Acknowledged and delete when age and disk threshold reached.
If auto-acknowledge is False:
If fileset type is SIG:
'S' -> 'N' -> 'A' progression
Else:
promote to 'A'
delete an 'A' fileset
'''
logger.info("(%s)Exceed %d days. Eligible to delete: %d" %(hex(deviceid),dmfileset['auto_trigger_age'],dmfilestats.count()))
if auto_acknowledge_enabled:
if dmfileset['type'] == dmactions_types.SIG:
logger.debug("Sig Proc Input Files auto acknowledge enabled")
'''
Do not require 'S' -> 'N' -> 'A' progression before action; mark 'A' and process
'''
A_list = dmfilestats.filter(action_state='A')
if A_list.count() > 0:
# there are filesets acknowledged and ready to be deleted. Covers situation where user has
#already acknowledged but recently enabled auto-acknowledge as well.
#deleteme = A_list[0]
try:
actiondmfilestat = getfirstnotpreserved(A_list, dmactions.DELETE)
logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName))
except DMExceptions.NoDMFileStat:
logger.info("(%s) No filesets to delete on this device" % hex(deviceid))
except:
logger.error(traceback.format_exc())
if actiondmfilestat == None:
# Select oldest fileset regardless if its 'L','S','N','A'. This covers situation where user
# recently enabled auto-acknowledge
try:
actiondmfilestat = getfirstnotpreserved(dmfilestats, dmactions.DELETE)
logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName))
except DMExceptions.NoDMFileStat:
logger.info("(%s) No filesets to delete on this device" % hex(deviceid))
except:
logger.error(traceback.format_exc())
else:
if dmfileset['type'] == dmactions_types.SIG:
logger.debug("Sig Proc Input Files auto acknowledge disabled")
'''
Need to select # of filesets and mark 'S'
Need to find 'S' filesets and notify (set to 'N')
Set queue length to 5
sum(S,N,A) == queue length
'''
#Get email recipient
try:
recipient = User.objects.get(username='******').email
recipient = recipient.replace(',',' ').replace(';',' ').split()
except:
recipient = None
# Need to select Experiments to process with appropriate dmfilestats action states.
exps = Experiment.objects.exclude(storage_options='KI').exclude(expDir='').order_by('date')
L_list = exps.filter(results_set__dmfilestat__in = dmfilestats.filter(action_state='L')).distinct()
S_list = exps.filter(results_set__dmfilestat__in = dmfilestats.filter(action_state='S')).distinct()
N_list = exps.filter(results_set__dmfilestat__in = dmfilestats.filter(action_state='N')).distinct()
A_list = exps.filter(results_set__dmfilestat__in = dmfilestats.filter(action_state='A')).distinct()
logger.info("Experiments (Keep=False) L:%d S:%d N:%d A:%d" % (L_list.count(),S_list.count(),N_list.count(),A_list.count()))
queue_length = 5
to_select = queue_length - (A_list.count() + N_list.count() + S_list.count())
if to_select > 0:
# Mark to_select number of oldest Experiments, from 'L', to 'S'
promoted = dmfilestats.filter(result__experiment__id__in=list(L_list[:to_select].values_list('id',flat=True)))
if auto_action_enabled:
promoted.update(action_state = 'S')
for dmfilestat in promoted:
EventLog.objects.add_entry(dmfilestat.result, "Signal Processing Input Selected for Deletion", username='******')
# Get updated list of Selected items
selected = dmfilestats.filter(action_state='S')
# Send Selected items to be notified
if selected.count() > 0 and auto_action_enabled:
logger.debug("notify recipient %s" % (recipient))
if notify([dmfilestat.result.resultsName for dmfilestat in selected],recipient):
selected.update(action_state = 'N')
for dmfilestat in selected:
EventLog.objects.add_entry(dmfilestat.result, "Notification for Deletion Sent", username='******')
try:
actiondmfilestat = getfirstnotpreserved(dmfilestats.filter(action_state='A'), dmactions.DELETE)
logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName))
except DMExceptions.NoDMFileStat:
logger.info("(%s) No filesets to delete on this device" % hex(deviceid))
except:
logger.error(traceback.format_exc())
else:
try:
actiondmfilestat = getfirstnotpreserved(dmfilestats, dmactions.DELETE)
logger.info("(%s)Picked: %s" % (hex(deviceid),actiondmfilestat.result.resultsName))
except DMExceptions.NoDMFileStat:
logger.info("(%s) No filesets to delete on this device" % hex(deviceid))
except:
logger.error(traceback.format_exc())
#Bail out if disabled
if auto_action_enabled != True:
logger.info("Data management auto-action is disabled.")
return
if actiondmfilestat is not None:
delete_action('dm_agent', user_comment, actiondmfilestat)
else:
logger.error("Unknown or unhandled action: %s" % dmfileset['auto_action'])
except (DMExceptions.FilePermission,
DMExceptions.InsufficientDiskSpace,
DMExceptions.MediaNotSet,
DMExceptions.MediaNotAvailable,
DMExceptions.FilesInUse) as e:
message = Message.objects.filter(tags__contains=e.tag)
if not message:
Message.error(e.message,tags=e.tag)
#TODO: TS-6525: This logentry will repeat for an Archive action every 30 seconds until the cause of the exception is fixed.
#at least, while the message banner is raised, suppress additional Log Entries.
EventLog.objects.add_entry(actiondmfilestat.result,"%s - %s" % (dmfileset['type'], e.message),username='******')
except DMExceptions.SrcDirDoesNotExist as e:
if actiondmfilestat.dmfileset.type == dmactions_types.SIG:
msg = "Src Dir not found: %s. Setting action_state to Deleted" % e.message
EventLog.objects.add_entry(actiondmfilestat.result,msg,username='******')
actiondmfilestat.setactionstate('DD')
logger.info(msg)
except Exception as inst:
msg = ''
if actiondmfilestat:
msg = "Auto-action error on %s " % actiondmfilestat.result.resultsName
msg += " Error: %s" % str(inst)
logger.exception("%s - %s" % (dmfileset['type'], msg))
if actiondmfilestat:
EventLog.objects.add_entry(actiondmfilestat.result,"%s - %s" % (dmfileset['type'], msg),username='******')
finally:
applock.unlock()
#logger.debug("(%05d,%05d) Release lock file" % (os.getppid(),os.getpid()))
return
def fileserver_space_check():
'''For each file server, compare disk usage to backup threshold value.
If disk usage exceeds threshold, launch celery task to delete/archive
raw data directories.
'''
# Get GlobalConfig object in order to access auto-acknowledge bit
gc = GlobalConfig.get()
auto_acknowledge = gc.auto_archive_ack
auto_action_enabled = gc.auto_archive_enable
# Get list of File Server objects
file_servers = FileServer.objects.all().order_by('pk').values()
# Get list of Report Storage objects
report_storages = ReportStorage.objects.all().order_by('pk').values()
# dict of fileset cateogries each with list of partition ids that can be acted upon.
category_list = {}
#-------------------------------------------------
# DELETE action only happens if threshold reached
#-------------------------------------------------
for dmfileset in DMFileSet.objects.filter(
version=settings.RELVERSION).filter(auto_action='DEL').values():
cat_name = slugify(dmfileset['type'])
category_list[cat_name] = {
'dmfileset': dmfileset,
'devlist': [],
'partitions': [],
}
for partition in _partitions(file_servers, report_storages):
if partition['diskusage'] >= dmfileset['auto_trigger_usage']:
logger.info("%s %s %.2f%% exceeds %s %.0f%%" %
(hex(partition['devid']), partition['path'],
partition['diskusage'], dmfileset['type'],
dmfileset['auto_trigger_usage']))
category_list[cat_name]['devlist'].append(partition['devid'])
category_list[cat_name]['partitions'].append(partition)
else:
logger.info("%s %s %.2f%% below %s %.0f%%" %
(hex(partition['devid']), partition['path'],
partition['diskusage'], dmfileset['type'],
dmfileset['auto_trigger_usage']))
# uniquify the deviceid list
category_list[cat_name]['devlist'] = list(
set(category_list[cat_name]['devlist']))
#-------------------------------------------------------------------------------
#ARCHIVE action happens as soon as grace period has expired (no threshold check)
#-------------------------------------------------------------------------------
for dmfileset in DMFileSet.objects.filter(
version=settings.RELVERSION).filter(auto_action='ARC').values():
cat_name = slugify(dmfileset['type'])
category_list[cat_name] = {
'dmfileset': dmfileset,
'devlist': [],
'partitions': [],
}
for partition in _partitions(file_servers, report_storages):
logger.debug("%s %s" %
(partition['path'], hex(partition['devid'])))
category_list[cat_name]['devlist'].append(partition['devid'])
category_list[cat_name]['partitions'].append(partition)
# uniquify the deviceid list
category_list[cat_name]['devlist'] = list(
set(category_list[cat_name]['devlist']))
#-------------------------------------------------------------
# Action loop - for each category, launch action per deviceid
#-------------------------------------------------------------
if not auto_action_enabled:
logger.info("Data management auto-action is disabled.")
# update any DMFileStats that are in use by active analysis jobs
try:
update_files_in_use()
except:
logger.error('Unable to update active DMFileStats')
logger.error(traceback.format_exc())
# Checks for manual export and archive requests.
manage_manual_action.delay()
for category, dict in category_list.iteritems():
for deviceid in dict['devlist']:
pathlist = [
item['path'] for item in dict['partitions']
if item['devid'] == deviceid
]
async_task_result = manage_data.delay(deviceid, dict['dmfileset'],
pathlist, auto_acknowledge,
auto_action_enabled)
return
def manage_data(deviceid, dmfileset, pathlist, auto_acknowledge_enabled,
auto_action_enabled):
logger.debug("manage_data: %s %s %s" %
(dmfileset['auto_action'], hex(deviceid), dmfileset['type']))
def getfirstnotpreserved(dmfilestats, action, threshdate):
'''QuerySet of DMFileStat objects. Returns first instance of an object
with preserve_data set to False and passes the action_validation test'''
logger.debug("Function: %s()" % sys._getframe().f_code.co_name)
logger.debug("Looking at %d dmfilestat objects" % dmfilestats.count())
for archiveme in dmfilestats:
if not archiveme.getpreserved():
try:
dmactions.action_validation(archiveme, action)
return archiveme
except (DMExceptions.FilesInUse, DMExceptions.FilesMarkedKeep):
logger.debug(
"%s Failed action_validation. Try next fileset" %
archiveme.result.resultsName)
except DMExceptions.BaseInputLinked:
# want to allow Basecalling Input delete if all results are expired
relatedObjs = DMFileStat.objects.filter(
result__experiment=archiveme.result.experiment,
dmfileset__type=dmactions_types.BASE)
if relatedObjs.count() == relatedObjs.filter(
created__lt=threshdate).count():
archiveme.allow_delete = True
return archiveme
else:
logger.debug(
"%s Failed action_validation. Try next fileset" %
archiveme.result.resultsName)
except:
logger.error(traceback.format_exc())
else:
logger.debug("Skipped a preserved fileset: %s" %
archiveme.result.resultsName)
logger.info("%d filestat objects are preserved." % dmfilestats.count())
raise DMExceptions.NoDMFileStat("NONE FOUND")
try:
#logger.debug("manage_data lock for %s (%d)" % (dmfileset['type'], os.getpid()))
#Create lock file to prevent more than one celery task for each process_type and partition
lock_id = "%s_%s" % (hex(deviceid), slugify(dmfileset['type']))
applock = TaskLock(lock_id)
if not (applock.lock()):
logger.info("Did not acquire lock file: %s" % lock_id)
return
logger.debug("Worker PID %d lock_id created %s" %
(os.getpid(), lock_id))
except Exception as e:
logger.exception(e)
try:
#---------------------------------------------------------------------------
# Database object filtering
#---------------------------------------------------------------------------
actiondmfilestat = None
user_comment = "Auto Action"
# Order by incrementing pk. This puts them in chronological order.
# Select DMFileStat objects of category DMFileSet.type (1/4th of all objects)
dmfilestats = DMFileStat.objects.filter(
dmfileset__type=dmfileset['type']).order_by('created')
tot_obj = dmfilestats.count()
# Select objects not yet processed
dmfilestats = dmfilestats.filter(action_state__in=['L', 'S', 'N', 'A'])
tot_act = dmfilestats.count()
logger.info("(%s)Total %s: %d Active: %d" %
(hex(deviceid), dmfileset['type'], tot_obj, tot_act))
# Select objects that are old enough
threshdate = datetime.now(
pytz.UTC) - timedelta(days=dmfileset['auto_trigger_age'])
dmfilestats = dmfilestats.filter(created__lt=threshdate)
logger.info(
"(%s)Total %s: %d Expired: %d" %
(hex(deviceid), dmfileset['type'], tot_obj, dmfilestats.count()))
# Select objects stored on the deviceid
query = Q()
for path in pathlist:
if dmfileset['type'] == dmactions_types.SIG:
query |= Q(result__experiment__expDir__startswith=path)
# These categories have files in both directory paths.
elif dmfileset['type'] in [
dmactions_types.INTR, dmactions_types.BASE
]:
query |= Q(result__experiment__expDir__startswith=path)
query |= Q(result__reportstorage__dirPath__startswith=path)
else:
query |= Q(result__reportstorage__dirPath__startswith=path)
dmfilestats = dmfilestats.filter(query)
logger.info("(%s)Total %s: %d On '%s' path: %d" %
(hex(deviceid), dmfileset['type'], tot_obj, path,
dmfilestats.count()))
# Exclude objects marked 'Keep' upfront to optimize db access
if dmfileset['type'] == dmactions_types.SIG:
dmfilestats = dmfilestats.exclude(
result__experiment__storage_options="KI")
else:
dmfilestats = dmfilestats.exclude(preserve_data=True)
logger.info(
"(%s)Total %s: %d Not Preserved: %d" %
(hex(deviceid), dmfileset['type'], tot_obj, dmfilestats.count()))
#---------------------------------------------------------------------------
# Archive
#---------------------------------------------------------------------------
if dmfileset['auto_action'] == 'ARC':
'''
Rules:
1) archive a fileset as soon as age threshold reached, regardless of disk threshold
2) select oldest fileset
3) select unprocessed fileset
4) Archiving does not require 'S' -> 'N' -> 'A' progression before action
5) Do not include filesets marked 'E' in auto-action - can get stuck on that fileset forever
'''
logger.info("(%s)Exceed %d days. Eligible to archive: %d" %
(hex(deviceid), dmfileset['auto_trigger_age'],
dmfilestats.count()))
actiondmfilestat = None
#Bail out if disabled
if auto_action_enabled != True:
logger.info("Data management auto-action is disabled.")
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id))
return
# Select first object stored on the deviceid
try:
actiondmfilestat = getfirstnotpreserved(
dmfilestats, dmactions.ARCHIVE, threshdate)
logger.info(
"(%s)Picked: %s" %
(hex(deviceid), actiondmfilestat.result.resultsName))
except DMExceptions.NoDMFileStat:
logger.debug("No filesets to archive on this device: %s" %
hex(deviceid))
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id))
except:
logger.error(traceback.format_exc())
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id))
else:
archive_action('dm_agent', user_comment, actiondmfilestat,
lock_id)
#---------------------------------------------------------------------------
# Delete
#---------------------------------------------------------------------------
elif dmfileset['auto_action'] == 'DEL':
'''
Rules: If auto-acknowledge is True:
promote to Acknowledged and delete when age and disk threshold reached.
If auto-acknowledge is False:
If fileset type is SIG:
'S' -> 'N' -> 'A' progression
Else:
promote to 'A'
delete an 'A' fileset
'''
logger.info("(%s)Exceed %d days. Eligible to delete: %d" %
(hex(deviceid), dmfileset['auto_trigger_age'],
dmfilestats.count()))
if auto_acknowledge_enabled:
if dmfileset['type'] == dmactions_types.SIG:
logger.debug(
"Sig Proc Input Files auto acknowledge enabled")
'''
Do not require 'S' -> 'N' -> 'A' progression before action; mark 'A' and process
'''
A_list = dmfilestats.filter(action_state='A')
if A_list.count() > 0:
# there are filesets acknowledged and ready to be deleted. Covers situation where user has
#already acknowledged but recently enabled auto-acknowledge as well.
#deleteme = A_list[0]
try:
actiondmfilestat = getfirstnotpreserved(
A_list, dmactions.DELETE, threshdate)
logger.info("(%s)Picked: %s" %
(hex(deviceid),
actiondmfilestat.result.resultsName))
except DMExceptions.NoDMFileStat:
logger.info(
"(%s) No filesets to delete on this device" %
hex(deviceid))
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id))
except:
logger.error(traceback.format_exc())
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id))
if actiondmfilestat == None:
# Select oldest fileset regardless if its 'L','S','N','A'. This covers situation where user
# recently enabled auto-acknowledge
try:
actiondmfilestat = getfirstnotpreserved(
dmfilestats, dmactions.DELETE, threshdate)
logger.info("(%s)Picked: %s" %
(hex(deviceid),
actiondmfilestat.result.resultsName))
except DMExceptions.NoDMFileStat:
logger.info(
"(%s) No filesets to delete on this device" %
hex(deviceid))
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id))
except:
logger.error(traceback.format_exc())
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id))
else:
if dmfileset['type'] == dmactions_types.SIG:
logger.debug(
"Sig Proc Input Files auto acknowledge disabled")
'''
Need to select # of filesets and mark 'S'
Need to find 'S' filesets and notify (set to 'N')
Set queue length to 5
sum(S,N,A) == queue length
'''
#Get email recipient
try:
recipient = User.objects.get(
username='******').email
recipient = recipient.replace(',', ' ').replace(
';', ' ').split()
except:
recipient = None
# Need to select Experiments to process with appropriate dmfilestats action states.
exps = Experiment.objects.exclude(
storage_options='KI').exclude(
expDir='').order_by('date')
L_list = exps.filter(
results_set__dmfilestat__in=dmfilestats.filter(
action_state='L')).distinct()
S_list = exps.filter(
results_set__dmfilestat__in=dmfilestats.filter(
action_state='S')).distinct()
N_list = exps.filter(
results_set__dmfilestat__in=dmfilestats.filter(
action_state='N')).distinct()
A_list = exps.filter(
results_set__dmfilestat__in=dmfilestats.filter(
action_state='A')).distinct()
logger.info(
"Experiments (Keep=False) L:%d S:%d N:%d A:%d" %
(L_list.count(), S_list.count(), N_list.count(),
A_list.count()))
queue_length = 5
to_select = queue_length - (
A_list.count() + N_list.count() + S_list.count())
if to_select > 0:
# Mark to_select number of oldest Experiments, from 'L', to 'S'
promoted = dmfilestats.filter(
result__experiment__id__in=list(
L_list[:to_select].values_list('id',
flat=True)))
if auto_action_enabled:
promoted.update(action_state='S')
for dmfilestat in promoted:
EventLog.objects.add_entry(
dmfilestat.result,
"Signal Processing Input Selected for Deletion",
username='******')
# Get updated list of Selected items
selected = dmfilestats.filter(action_state='S')
# Send Selected items to be notified
if selected.count() > 0 and auto_action_enabled:
logger.debug("notify recipient %s" % (recipient))
if notify([
dmfilestat.result.resultsName
for dmfilestat in selected
], recipient):
selected.update(action_state='N')
for dmfilestat in selected:
EventLog.objects.add_entry(
dmfilestat.result,
"Notification for Deletion Sent",
username='******')
try:
actiondmfilestat = getfirstnotpreserved(
dmfilestats.filter(action_state='A'),
dmactions.DELETE, threshdate)
logger.info("(%s)Picked: %s" %
(hex(deviceid),
actiondmfilestat.result.resultsName))
except DMExceptions.NoDMFileStat:
logger.info(
"(%s) No filesets to delete on this device" %
hex(deviceid))
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id))
except:
logger.error(traceback.format_exc())
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id))
else:
try:
actiondmfilestat = getfirstnotpreserved(
dmfilestats, dmactions.DELETE, threshdate)
logger.info("(%s)Picked: %s" %
(hex(deviceid),
actiondmfilestat.result.resultsName))
except DMExceptions.NoDMFileStat:
logger.info(
"(%s) No filesets to delete on this device" %
hex(deviceid))
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id))
except:
logger.error(traceback.format_exc())
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id))
#Bail out if disabled
if auto_action_enabled != True:
logger.info("Data management auto-action is disabled.")
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id))
return
if actiondmfilestat is not None:
delete_action('dm_agent',
user_comment,
actiondmfilestat,
lock_id,
confirmed=getattr(actiondmfilestat,
'allow_delete', False))
else:
logger.error("Unknown or unhandled action: %s" %
dmfileset['auto_action'])
applock.unlock()
except (DMExceptions.FilePermission, DMExceptions.InsufficientDiskSpace,
DMExceptions.MediaNotSet, DMExceptions.MediaNotAvailable,
DMExceptions.FilesInUse) as e:
applock.unlock()
message = Message.objects.filter(tags__contains=e.tag)
if not message:
Message.error(e.message, tags=e.tag)
#TODO: TS-6525: This logentry will repeat for an Archive action every 30 seconds until the cause of the exception is fixed.
#at least, while the message banner is raised, suppress additional Log Entries.
EventLog.objects.add_entry(actiondmfilestat.result,
"%s - %s" %
(dmfileset['type'], e.message),
username='******')
except DMExceptions.SrcDirDoesNotExist as e:
applock.unlock()
msg = "Src Dir not found: %s. Setting action_state to Deleted" % e.message
EventLog.objects.add_entry(actiondmfilestat.result,
msg,
username='******')
actiondmfilestat.setactionstate('DD')
logger.info(msg)
except Exception as inst:
applock.unlock()
msg = ''
if actiondmfilestat:
msg = "Auto-action error on %s " % actiondmfilestat.result.resultsName
msg += " Error: %s" % str(inst)
logger.exception("%s - %s" % (dmfileset['type'], msg))
if actiondmfilestat:
EventLog.objects.add_entry(actiondmfilestat.result,
"%s - %s" % (dmfileset['type'], msg),
username='******')
return
