def msg_banner(name,
categories,
logfile,
status,
importReport=None,
result=None,
exp=None):
msg = '(%s) Import %s, %s' % (name, ', '.join(categories), status)
if status == 'Completed':
if importReport:
logurl = reverse('dm_log', args=(result.pk, ))
msg += " <a href='%s' data-toggle='modal' data-target='#modal_report_log'>View Report Log</a>" % (
logurl)
reporturl = reverse('report', args=(result.pk, ))
msg += " Imported Report: <a href='%s'>%s</a>'" % (
reporturl, result.resultsName)
else:
msg += " Imported Run available for analysis: <a href='/data/'>%s</a>" % exp.expName
Message.objects.filter(tags=logfile).delete()
if status == 'Error':
errlog = reverse('import_data_log', args=(logfile, ))
msg += " <a href='%s' data-toggle='modal' data-target='#modal_report_log'>Error Log</a>" % (
errlog)
Message.error(msg, tags=logfile)
else:
Message.info(msg, tags=logfile)
def msg_banner(status, importing, error_str=""):
msg = "(%s) Import %s, %s." % (importing.name, importing.selected_str,
status)
if "Completed" in status:
if importing.result:
logurl = "/data/datamanagement/log/%s/" % importing.result.pk
msg += (
" <a href='%s' data-toggle='modal' data-target='#modal_report_log'>View Report Log</a>"
% (logurl))
if dmactions_types.OUT in importing.dmtypes:
reporturl = "/report/%s/" % importing.result.pk
msg += " Imported Report: <a href='%s'>%s</a>'" % (
reporturl,
importing.result.resultsName,
)
else:
msg += (
" Imported Run available for analysis: <a href='/data/'>%s</a>"
% importing.exp.expName)
Message.objects.filter(tags=importing.tag).delete()
if status == "ERROR":
errlog = "/data/datamanagement/import_data_log/%s" % importing.logfile.name
msg += (
" %s <a href='%s' data-toggle='modal' data-target='#modal_report_log'>Error Log</a>"
% (error_str, errlog))
Message.error(msg, tags=importing.tag, route=importing.user)
else:
Message.info(msg, tags=importing.tag, route=importing.user)
def registration(request):
if request.method == 'POST': # If the form has been submitted...
form = UserRegistrationForm(request.POST)
if form.is_valid(): # All validation rules pass
# create user from cleaned_data
username = form.cleaned_data['username']
email = form.cleaned_data['email']
password = form.cleaned_data['password1']
new_user = User.objects.create_user(username, email, password)
new_user.is_active = False # Users created inactive by default
# Add users to ionusers group, for a default set of permissions
try:
group = Group.objects.get(name='ionusers')
new_user.groups.add(group)
except Group.DoesNotExist:
logger.warn("Group ionusers not found. " +
"New user %s will lack permission to do anything beyond look!", username)
new_user.save()
# Send global message notifying of pending registration
msg = "New pending user registration for '%s'. " + \
"Please visit <a href='%s'>Account Management</a> (as an admin user) to review."
Message.warn(msg % (username, urlresolvers.reverse('configure_account')))
# Otherwise redirect to a success page. Awaiting approval
return shortcuts.redirect(urlresolvers.reverse('signup_pending'))
else:
# Blank Form
form = UserRegistrationForm()
context = template.RequestContext(request, {'form': form, })
return shortcuts.render_to_response("rundb/login/ion_account_reg.html",
context_instance=context)
def registration(request):
if request.method == 'POST': # If the form has been submitted...
form = UserRegistrationForm(request.POST)
if form.is_valid(): # All validation rules pass
# create user from cleaned_data
username = form.cleaned_data['username']
email = form.cleaned_data['email']
password = form.cleaned_data['password1']
new_user = User.objects.create_user(username, email, password)
new_user.is_active = False # Users created inactive by default
# Add users to ionusers group, for a default set of permissions
try:
group = Group.objects.get(name='ionusers')
new_user.groups.add(group)
except Group.DoesNotExist:
logger.warn("Group ionusers not found. " +
"New user %s will lack permission to do anything beyond look!", username)
new_user.save()
# Send global message notifying of pending registration
msg = "New pending user registration for '%s'. " + \
"Please visit <a href='%s'>Account Management</a> (as an admin user) to review."
Message.warn(msg % (username, urlresolvers.reverse('configure_account')))
# Otherwise redirect to a success page. Awaiting approval
return shortcuts.redirect(urlresolvers.reverse('signup_pending'))
else:
# Blank Form
form = UserRegistrationForm()
context = template.RequestContext(request, {'form': form, })
return shortcuts.render_to_response("rundb/login/ion_account_reg.html",
context_instance=context)
def update_banner(alerts):
new = False
# update message banner
message = Message.objects.filter(tags="service_status_alert")
if len(alerts) > 0:
msg = 'ALERT system services are down: %s. ' % ', '.join(alerts)
msg += ' Please contact your system administrator for assistance.'
if not message or message[0].body != msg:
message.delete()
Message.warn(msg, tags="service_status_alert")
new = True
else:
message.delete()
print '...updated message banner'
return new
def update_banner(alerts):
new = False
# update message banner
message = Message.objects.filter(tags="service_status_alert")
if len(alerts) > 0:
msg = 'ALERT system services are down: %s. ' % ', '.join(alerts)
msg += ' Please contact your system administrator for assistance.'
if not message or message[0].body != msg:
message.delete()
Message.warn(msg, tags="service_status_alert")
new = True
else:
message.delete()
print '...updated message banner'
return new
def registration(request):
if request.method == "POST": # If the form has been submitted...
form = UserRegistrationForm(request.POST)
if form.is_valid(): # All validation rules pass
# create user from cleaned_data
username = form.cleaned_data["username"]
email = form.cleaned_data["email"]
password = form.cleaned_data["password1"]
new_user = User.objects.create_user(username, email, password)
new_user.is_active = False # Users created inactive by default
# Add users to ionusers group, for a default set of permissions
try:
group = Group.objects.get(name="ionusers")
new_user.groups.add(group)
except Group.DoesNotExist:
logger.warn(
"Group ionusers not found. " +
"New user %s will lack permission to do anything beyond look!",
username,
)
new_user.save()
# update UserProfile to indicate this account needs activation
new_user.userprofile.needs_activation = True
new_user.userprofile.save()
# Send global message notifying of pending registration
msg = (
"New pending user registration for '%s'. " +
"Please visit <a href='%s'>Account Management</a> to review."
) # TODO: i18n ?
Message.warn(
msg % (username, urlresolvers.reverse("configure_account")),
route=Message.USER_STAFF,
)
# Otherwise redirect to a success page. Awaiting approval
return shortcuts.redirect(urlresolvers.reverse("signup_pending"))
else:
# Blank Form
form = UserRegistrationForm()
context = template.RequestContext(request, {"form": form})
return shortcuts.render_to_response("rundb/login/ion_account_reg.html",
context_instance=context)
def msg_banner(name, categories, logfile, status, importReport=None, result=None, exp=None):
msg = '(%s) Import %s, %s' % (name, ', '.join(categories), status)
if status == 'Completed':
if importReport:
logurl = reverse('dm_log', args=(result.pk,))
msg += " <a href='%s' data-toggle='modal' data-target='#modal_report_log'>View Report Log</a>" % (logurl)
reporturl = reverse('report', args=(result.pk,))
msg += " Imported Report: <a href='%s'>%s</a>'" % (reporturl, result.resultsName)
else:
msg += " Imported Run available for analysis: <a href='/data/'>%s</a>" % exp.expName
Message.objects.filter(tags=logfile).delete()
if status == 'Error':
errlog = reverse('import_data_log', args=(logfile,))
msg += " <a href='%s' data-toggle='modal' data-target='#modal_report_log'>Error Log</a>" % (errlog)
Message.error(msg, tags=logfile)
else:
Message.info(msg, tags=logfile)
def msg_banner(status, importing, result=None, exp=None, msg_str=''):
msg = '(%s) Import %s, %s.' % (importing.name, importing.selected_str, status)
if 'Completed' in status:
if result:
logurl = reverse('dm_log', args=(result.pk,))
msg += " <a href='%s' data-toggle='modal' data-target='#modal_report_log'>View Report Log</a>" % (logurl)
if dmactions_types.OUT in importing.selected_str:
reporturl = reverse('report', args=(result.pk,))
msg += " Imported Report: <a href='%s'>%s</a>'" % (reporturl, result.resultsName)
else:
msg += " Imported Run available for analysis: <a href='/data/'>%s</a>" % exp.expName
Message.objects.filter(tags=importing.tag).delete()
if status == 'Error':
errlog = reverse('import_data_log', args=(importing.logfile.name,))
msg += ": %s <a href='%s' data-toggle='modal' data-target='#modal_report_log'>Error Log</a>" % (msg_str, errlog)
Message.error(msg, tags=importing.tag, route=importing.user)
else:
Message.info(msg, tags=importing.tag, route=importing.user)
def check_raid_status():
# check RAID state and alert user with message banner of any problems
from iondb.rundb.models import Message
async_result = get_raid_stats_json.apply_async(queue="periodic")
raidinfo = async_result.get(timeout=600)
if async_result.failed():
logger.debug("Failed getting RAID info.")
return
raid_status = raid_utils.get_raid_status(raidinfo)
if len(raid_status) > 0:
message = Message.objects.filter(tags="raid_alert")
# show alert for primary internal storage RAID
if raid_utils.ERROR in [r.get('status') for r in raid_status]:
if not message:
msg = 'WARNING: RAID storage disk error.'
golink = "<a href='%s' > Visit Services Tab </a>" % ('/configure/services/')
Message.warn(msg+" %s" % golink,tags="raid_alert")
else:
message.delete()
def msg_banner(status, importing, error_str=''):
msg = '(%s) Import %s, %s.' % (importing.name, importing.selected_str, status)
if 'Completed' in status:
if importing.result:
logurl = '/data/datamanagement/log/%s/' % importing.result.pk
msg += " <a href='%s' data-toggle='modal' data-target='#modal_report_log'>View Report Log</a>" % (
logurl)
if dmactions_types.OUT in importing.dmtypes:
reporturl = '/report/%s/' % importing.result.pk
msg += " Imported Report: <a href='%s'>%s</a>'" % (reporturl, importing.result.resultsName)
else:
msg += " Imported Run available for analysis: <a href='/data/'>%s</a>" % importing.exp.expName
Message.objects.filter(tags=importing.tag).delete()
if status == 'ERROR':
errlog = '/data/datamanagement/import_data_log/%s' % importing.logfile.name
msg += " %s <a href='%s' data-toggle='modal' data-target='#modal_report_log'>Error Log</a>" % (
error_str, errlog)
Message.error(msg, tags=importing.tag, route=importing.user)
else:
Message.info(msg, tags=importing.tag, route=importing.user)
def msg_banner(status, importing, result=None, exp=None, msg_str=''):
msg = '(%s) Import %s, %s.' % (importing.name, importing.selected_str,
status)
if 'Completed' in status:
if result:
logurl = '/data/datamanagement/log/%s/' % result.pk
msg += " <a href='%s' data-toggle='modal' data-target='#modal_report_log'>View Report Log</a>" % (
logurl)
if dmactions_types.OUT in importing.selected_str:
reporturl = '/report/%s/' % result.pk
msg += " Imported Report: <a href='%s'>%s</a>'" % (
reporturl, result.resultsName)
else:
msg += " Imported Run available for analysis: <a href='/data/'>%s</a>" % exp.expName
Message.objects.filter(tags=importing.tag).delete()
if status == 'ERROR':
errlog = '/data/datamanagement/import_data_log/%s' % importing.logfile.name
msg += " %s <a href='%s' data-toggle='modal' data-target='#modal_report_log'>Error Log</a>" % (
msg_str, errlog)
Message.error(msg, tags=importing.tag, route=importing.user)
else:
Message.info(msg, tags=importing.tag, route=importing.user)
def manage_manual_action():
logger.debug("manage_manual_action")
try:
#Create lock file to prevent more than one celery task for manual action (export or archive)
lockfile = '/var/run/celery/manual_action.lock'
applock = ApplicationLock(lockfile)
if not(applock.lock()):
logger.debug("failed to acquire lock file: %d" % os.getpid())
logger.info("manage_manual_action task still executing")
return
logger.info("Worker PID %d lockfile created %s" % (os.getpid(),lockfile))
except Exception as e:
logger.exception(e)
try:
#
# Check for manually selected Archive and Export actions - action_state == 'SA' or 'SE'
# These jobs should execute even when auto action is disabled.
# Note: manual actions will not be executed in the order they are selected, but by age.
#
user_comment = "Manual Action"
manualSelects = DMFileStat.objects.filter(action_state__in=['SA','SE']).order_by('created')
if manualSelects.exists():
actiondmfilestat = manualSelects[0]
dmfileset = actiondmfilestat.dmfileset
project_msg = {}
msg_dict = {}
msg_dict[dmfileset.type] = "Success"
project_msg[actiondmfilestat.result_id] = msg_dict
if actiondmfilestat.action_state == 'SA':
logger.info("Manual Archive Action: %s from %s" % (dmfileset.type,actiondmfilestat.result.resultsName))
archive_action('dm_agent', user_comment, actiondmfilestat)
datatasks.project_msg_banner('', project_msg, dmactions.ARCHIVE)
elif actiondmfilestat.action_state == 'SE':
logger.info("Manual Export Action: %s from %s" % (dmfileset.type,actiondmfilestat.result.resultsName))
export_action('dm_agent', user_comment, actiondmfilestat)
datatasks.project_msg_banner('', project_msg, dmactions.EXPORT)
else:
logger.warn("Dev Error: we don't handle this '%s' here" % actiondmfilestat.action_state)
return
#NOTE: all these exceptions are also handled in manage_data() below. beaucoup de duplication de code
except (DMExceptions.FilePermission,
DMExceptions.InsufficientDiskSpace,
DMExceptions.MediaNotSet,
DMExceptions.MediaNotAvailable,
DMExceptions.FilesInUse) as e:
message = Message.objects.filter(tags__contains=e.tag)
if not message:
Message.error(e.message,tags=e.tag)
#TODO: TS-6525: This logentry will repeat for an Archive action every 30 seconds until the cause of the exception is fixed.
#at least, while the message banner is raised, suppress additional Log Entries.
dmactions.add_eventlog(actiondmfilestat, "%s - %s" % (dmfileset.type, e.message), username='******')
# Revert this dmfilestat object action-state to Local
actiondmfilestat.setactionstate('L')
except DMExceptions.SrcDirDoesNotExist as e:
if actiondmfilestat.dmfileset.type == dmactions_types.SIG:
msg = "Src Dir not found: %s. Setting action_state to Deleted" % e.message
EventLog.objects.add_entry(actiondmfilestat.result,msg,username='******')
actiondmfilestat.setactionstate('DD')
logger.info(msg)
except Exception as e:
msg = "action error on %s " % actiondmfilestat.result.resultsName
msg += " Error: %s" % str(e)
logger.exception("%s - %s" % (dmfileset.type, msg))
dmactions.add_eventlog(actiondmfilestat,"%s - %s" % (dmfileset.type, msg),username='******')
# Revert this dmfilestat object action-state to Local
actiondmfilestat.setactionstate('L')
finally:
applock.unlock()
logger.debug("(%05d,%05d) Release lock file" % (os.getppid(),os.getpid()))
return
def manage_manual_action():
logger.debug("manage_manual_action", extra = {'logid':"%s" % ('manual_action')})
try:
#Create lock file to prevent more than one celery task for manual action (export or archive)
lock_id = 'manual_action_lock_id'
d = {'logid':"%s" % (lock_id)}
applock = TaskLock(lock_id)
if not(applock.lock()):
logger.debug("lock file exists: %s(%s)" % (lock_id, applock.get()), extra = d)
return
logger.debug("lock file created: %s(%s)" % (lock_id, applock.get()), extra = d)
except Exception as e:
logger.error(traceback.format_exc(), extra = d)
try:
#
# Check for manually selected Archive and Export actions - action_state == 'SA' or 'SE' or 'SD'
# Manual Delete actions do not get processed here. Only suspended Delete actions get processed here.
# These jobs should execute even when auto action is disabled.
# Note: manual actions will not be executed in the order they are selected, but by age.
#
manualSelects = DMFileStat.objects.filter(action_state__in=['SA','SE','SD']).order_by('created')
if manualSelects.exists():
actiondmfilestat = manualSelects[0]
user = actiondmfilestat.user_comment.get('user', 'dm_agent')
user_comment = actiondmfilestat.user_comment.get('user_comment', 'Manual Action')
dmfileset = actiondmfilestat.dmfileset
applock.update(actiondmfilestat.result.resultsName)
if actiondmfilestat.action_state == 'SA':
logger.info("Manual Archive Action: %s from %s" % (dmfileset.type,actiondmfilestat.result.resultsName), extra = d)
archive_action(user, user_comment, actiondmfilestat, lock_id, msg_banner = True)
elif actiondmfilestat.action_state == 'SE':
logger.info("Manual Export Action: %s from %s" % (dmfileset.type,actiondmfilestat.result.resultsName), extra = d)
export_action(user, user_comment, actiondmfilestat, lock_id, msg_banner = True)
elif actiondmfilestat.action_state == 'SD':
logger.info("Delete Action: %s from %s" % (dmfileset.type,actiondmfilestat.result.resultsName), extra = d)
delete_action(user, "Continuing delete action after being suspended", actiondmfilestat, lock_id, msg_banner = True)
else:
logger.warn("Dev Error: we don't handle this '%s' here" % actiondmfilestat.action_state, extra = d)
else:
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed on exit %s" % (os.getpid(),lock_id), extra = d)
#NOTE: all these exceptions are also handled in manage_data() below. beaucoup de duplication de code
except (DMExceptions.FilePermission,
DMExceptions.InsufficientDiskSpace,
DMExceptions.MediaNotSet,
DMExceptions.MediaNotAvailable,
DMExceptions.FilesInUse) as e:
applock.unlock()
message = Message.objects.filter(tags__contains=e.tag)
if not message:
Message.error(e.message,tags=e.tag)
#TODO: TS-6525: This logentry will repeat for an Archive action every 30 seconds until the cause of the exception is fixed.
#at least, while the message banner is raised, suppress additional Log Entries.
dmactions.add_eventlog(actiondmfilestat, "%s - %s" % (dmfileset.type, e.message), username='******')
# Revert this dmfilestat object action-state to Local
actiondmfilestat.setactionstate('L')
except DMExceptions.SrcDirDoesNotExist as e:
applock.unlock()
msg = "Src Dir not found: %s. Setting action_state to Deleted" % e.message
EventLog.objects.add_entry(actiondmfilestat.result,msg,username='******')
actiondmfilestat.setactionstate('DD')
logger.info(msg, extra = d)
except Exception as e:
applock.unlock()
msg = "action error on %s " % actiondmfilestat.result.resultsName
msg += " Error: %s" % str(e)
logger.error("%s - %s" % (dmfileset.type, msg), extra = d)
logger.error(traceback.format_exc(), extra = d)
dmactions.add_eventlog(actiondmfilestat,"%s - %s" % (dmfileset.type, msg),username='******')
# Revert this dmfilestat object action-state to Local
actiondmfilestat.setactionstate('L')
return
def manage_manual_action():
logger.debug("manage_manual_action")
try:
#Create lock file to prevent more than one celery task for manual action (export or archive)
lock_id = 'manual_action_lock_id'
applock = TaskLock(lock_id)
if not (applock.lock()):
logger.debug("failed to acquire lock file: %d" % os.getpid())
logger.info("manage_manual_action task still executing")
return
logger.debug("Worker PID %d lock_id created %s" %
(os.getpid(), lock_id))
except Exception as e:
logger.exception(e)
try:
#
# Check for manually selected Archive and Export actions - action_state == 'SA' or 'SE' or 'SD'
# Manual Delete actions do not get processed here. Only suspended Delete actions get processed here.
# These jobs should execute even when auto action is disabled.
# Note: manual actions will not be executed in the order they are selected, but by age.
#
manualSelects = DMFileStat.objects.filter(
action_state__in=['SA', 'SE', 'SD']).order_by('created')
if manualSelects.exists():
actiondmfilestat = manualSelects[0]
user = actiondmfilestat.user_comment.get('user', 'dm_agent')
user_comment = actiondmfilestat.user_comment.get(
'user_comment', 'Manual Action')
dmfileset = actiondmfilestat.dmfileset
if actiondmfilestat.action_state == 'SA':
logger.info(
"Manual Archive Action: %s from %s" %
(dmfileset.type, actiondmfilestat.result.resultsName))
archive_action(user,
user_comment,
actiondmfilestat,
lock_id,
msg_banner=True)
elif actiondmfilestat.action_state == 'SE':
logger.info(
"Manual Export Action: %s from %s" %
(dmfileset.type, actiondmfilestat.result.resultsName))
export_action(user,
user_comment,
actiondmfilestat,
lock_id,
msg_banner=True)
elif actiondmfilestat.action_state == 'SD':
logger.info(
"Delete Action: %s from %s" %
(dmfileset.type, actiondmfilestat.result.resultsName))
delete_action(user,
"Continuing delete action after being suspended",
actiondmfilestat,
lock_id,
msg_banner=True)
else:
logger.warn("Dev Error: we don't handle this '%s' here" %
actiondmfilestat.action_state)
else:
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed on exit %s" %
(os.getpid(), lock_id))
#NOTE: all these exceptions are also handled in manage_data() below. beaucoup de duplication de code
except (DMExceptions.FilePermission, DMExceptions.InsufficientDiskSpace,
DMExceptions.MediaNotSet, DMExceptions.MediaNotAvailable,
DMExceptions.FilesInUse) as e:
applock.unlock()
message = Message.objects.filter(tags__contains=e.tag)
if not message:
Message.error(e.message, tags=e.tag)
#TODO: TS-6525: This logentry will repeat for an Archive action every 30 seconds until the cause of the exception is fixed.
#at least, while the message banner is raised, suppress additional Log Entries.
dmactions.add_eventlog(actiondmfilestat,
"%s - %s" % (dmfileset.type, e.message),
username='******')
# Revert this dmfilestat object action-state to Local
actiondmfilestat.setactionstate('L')
except DMExceptions.SrcDirDoesNotExist as e:
applock.unlock()
msg = "Src Dir not found: %s. Setting action_state to Deleted" % e.message
EventLog.objects.add_entry(actiondmfilestat.result,
msg,
username='******')
actiondmfilestat.setactionstate('DD')
logger.info(msg)
except Exception as e:
applock.unlock()
msg = "action error on %s " % actiondmfilestat.result.resultsName
msg += " Error: %s" % str(e)
logger.exception("%s - %s" % (dmfileset.type, msg))
dmactions.add_eventlog(actiondmfilestat,
"%s - %s" % (dmfileset.type, msg),
username='******')
# Revert this dmfilestat object action-state to Local
actiondmfilestat.setactionstate('L')
return
#!/usr/bin python
# Copyright (C) 2011 Ion Torrent Systems, Inc. All Rights Reserved
import argparse
from djangoinit import *
from iondb.rundb.models import Message
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Post a site message to the Torrent browser.")
parser.add_argument("message",
type=str,
help="The message string to be displayed.")
parser.add_argument("-d",
"--duplicate",
action="store_true",
help="Create a duplicate post if necessary.")
args = parser.parse_args()
msg = Message.objects.filter(body=args.message).count()
if not msg or args.duplicate:
Message.info(args.message)
print("Posted message")
else:
print("Didn't post duplicate message")
def manage_data(deviceid, dmfileset, pathlist, auto_acknowledge_enabled, auto_action_enabled):
logid = {'logid': "%s" % ('manage_data')}
logger.debug("manage_data: %s %s" % (dmfileset['auto_action'], dmfileset['type']), extra=logid)
def getfirstnotpreserved(dmfilestats, action, threshdate):
'''QuerySet of DMFileStat objects. Returns first instance of an object
with preserve_data set to False and passes the action_validation test'''
logger.debug("Function: %s()" % sys._getframe().f_code.co_name, extra=logid)
logger.debug("Looking at %d dmfilestat objects" % dmfilestats.count(), extra=logid)
for archiveme in dmfilestats:
if not archiveme.getpreserved():
try:
dmactions.action_validation(archiveme, action)
return archiveme
except(DMExceptions.FilesInUse, DMExceptions.FilesMarkedKeep):
logger.debug("%s Failed action_validation. Try next fileset" %
archiveme.result.resultsName, extra=logid)
except DMExceptions.BaseInputLinked:
# want to allow Basecalling Input delete if all results are expired
related_objs = DMFileStat.objects.filter(
result__experiment=archiveme.result.experiment, dmfileset__type=dmactions_types.BASE)
if related_objs.count() == related_objs.filter(created__lt=threshdate).count():
archiveme.allow_delete = True
return archiveme
else:
logger.debug("%s Failed action_validation. Try next fileset" %
archiveme.result.resultsName, extra=logid)
except:
logger.error(traceback.format_exc(), extra=logid)
else:
logger.debug("Skipped a preserved fileset: %s" % archiveme.result.resultsName, extra=logid)
logger.info("%d filestat objects are preserved." % dmfilestats.count(), extra=logid)
raise DMExceptions.NoDMFileStat("NONE FOUND")
try:
# logger.debug("manage_data lock for %s (%d)" % (dmfileset['type'], os.getpid()), extra = logid)
# Create lock file to prevent more than one celery task for each process_type and partition
lock_id = "%s_%s" % (hex(deviceid), slugify(dmfileset['type']))
logid = {'logid': "%s" % (lock_id)}
applock = TaskLock(lock_id)
if not applock.lock():
logger.info("Currently processing: %s(%s)" % (lock_id, applock.get()), extra=logid)
return
logger.debug("lock file created: %s(%s)" % (lock_id, applock.get()), extra=logid)
except Exception as e:
logger.error(traceback.format_exc(), extra=logid)
try:
#---------------------------------------------------------------------------
# Database object filtering
#---------------------------------------------------------------------------
actiondmfilestat = None
user_comment = "Auto Action"
# Order by incrementing pk. This puts them in chronological order.
# Select DMFileStat objects of category DMFileSet.type (1/4th of all objects)
dmfilestats = DMFileStat.objects.filter(dmfileset__type=dmfileset['type']).order_by('created')
tot_obj = dmfilestats.count()
# Select objects not yet processed
dmfilestats = dmfilestats.filter(action_state__in=['L', 'S', 'N', 'A'])
tot_act = dmfilestats.count()
# Select objects that are old enough
threshdate = datetime.now(pytz.UTC) - timedelta(days=dmfileset['auto_trigger_age'])
dmfilestats = dmfilestats.filter(created__lt=threshdate)
tot_exp = dmfilestats.count()
# Select objects stored on the deviceid
query = Q()
for path in pathlist:
if dmfileset['type'] == dmactions_types.SIG:
query |= Q(result__experiment__expDir__startswith=path)
# These categories have files in both directory paths.
elif dmfileset['type'] in [dmactions_types.INTR, dmactions_types.BASE]:
query |= Q(result__experiment__expDir__startswith=path)
query |= Q(result__reportstorage__dirPath__startswith=path)
else:
query |= Q(result__reportstorage__dirPath__startswith=path)
dmfilestats = dmfilestats.filter(query)
tot_onpath = dmfilestats.count()
# Exclude objects marked 'Keep' upfront to optimize db access
if dmfileset['type'] == dmactions_types.SIG:
dmfilestats = dmfilestats.exclude(result__experiment__storage_options="KI")
else:
dmfilestats = dmfilestats.exclude(preserve_data=True)
tot_notpreserved = dmfilestats.count()
# Compress to single log entry
logger.info("Total: %d Active: %d Expired: %d On Path: %d Not Preserved: %d" %
(tot_obj, tot_act, tot_exp, tot_onpath, tot_notpreserved), extra=logid)
#---------------------------------------------------------------------------
# Archive
#---------------------------------------------------------------------------
if dmfileset['auto_action'] == 'ARC':
'''
Rules:
1) archive a fileset as soon as age threshold reached, regardless of disk threshold
2) select oldest fileset
3) select unprocessed fileset
4) Archiving does not require 'S' -> 'N' -> 'A' progression before action
5) Do not include filesets marked 'E' in auto-action - can get stuck on that fileset forever
'''
logger.info("Exceed %d days. Eligible to archive: %d" %
(dmfileset['auto_trigger_age'], dmfilestats.count()), extra=logid)
actiondmfilestat = None
# Bail out if disabled
if auto_action_enabled != True:
logger.info("Data management auto-action is disabled.", extra=logid)
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id), extra=logid)
return
# Select first object stored on the deviceid
try:
actiondmfilestat = getfirstnotpreserved(dmfilestats, dmactions.ARCHIVE, threshdate)
logger.info("Picked: %s" % (actiondmfilestat.result.resultsName), extra=logid)
except DMExceptions.NoDMFileStat:
logger.debug("No filesets to archive on this device", extra=logid)
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id), extra=logid)
except:
logger.error(traceback.format_exc(), extra=logid)
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id), extra=logid)
else:
applock.update(actiondmfilestat.result.resultsName)
archive_action('dm_agent', user_comment, actiondmfilestat, lock_id)
#---------------------------------------------------------------------------
# Delete
#---------------------------------------------------------------------------
elif dmfileset['auto_action'] == 'DEL':
'''
Rules: If auto-acknowledge is True:
promote to Acknowledged and delete when age and disk threshold reached.
If auto-acknowledge is False:
If fileset type is SIG:
'S' -> 'N' -> 'A' progression
Else:
promote to 'A'
delete an 'A' fileset
'''
logger.info("Exceed %d days. Eligible to delete: %d" %
(dmfileset['auto_trigger_age'], dmfilestats.count()), extra=logid)
if auto_acknowledge_enabled:
if dmfileset['type'] == dmactions_types.SIG:
logger.debug("Sig Proc Input Files auto acknowledge enabled", extra=logid)
'''
Do not require 'S' -> 'N' -> 'A' progression before action; mark 'A' and process
'''
a_list = dmfilestats.filter(action_state='A')
if a_list.count() > 0:
# there are filesets acknowledged and ready to be deleted. Covers situation where user has
# already acknowledged but recently enabled auto-acknowledge as well.
# deleteme = a_list[0]
try:
actiondmfilestat = getfirstnotpreserved(a_list, dmactions.DELETE, threshdate)
logger.info("Picked: %s" % (actiondmfilestat.result.resultsName), extra=logid)
except DMExceptions.NoDMFileStat:
logger.info("No filesets to delete on this device", extra=logid)
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id), extra=logid)
except:
logger.error(traceback.format_exc(), extra=logid)
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id), extra=logid)
if actiondmfilestat == None:
# Select oldest fileset regardless if its 'L','S','N','A'. This covers situation where user
# recently enabled auto-acknowledge
try:
actiondmfilestat = getfirstnotpreserved(dmfilestats, dmactions.DELETE, threshdate)
logger.info("Picked: %s" % (actiondmfilestat.result.resultsName), extra=logid)
except DMExceptions.NoDMFileStat:
logger.info("No filesets to delete on this device", extra=logid)
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id), extra=logid)
except:
logger.error(traceback.format_exc(), extra=logid)
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id), extra=logid)
else:
if dmfileset['type'] == dmactions_types.SIG:
logger.debug("Sig Proc Input Files auto acknowledge disabled", extra=logid)
'''
Need to select # of filesets and mark 'S'
Need to find 'S' filesets and notify (set to 'N')
Set queue length to 5
sum(S,N,A) == queue length
'''
# Get email recipient
try:
recipient = User.objects.get(username='******').email
except:
recipient = None
# Need to select Experiments to process with appropriate dmfilestats action states.
exps = Experiment.objects.exclude(
storage_options='KI').exclude(expDir='').order_by('date')
l_list = exps.filter(
results_set__dmfilestat__in=dmfilestats.filter(action_state='L')).distinct()
s_list = exps.filter(
results_set__dmfilestat__in=dmfilestats.filter(action_state='S')).distinct()
n_list = exps.filter(
results_set__dmfilestat__in=dmfilestats.filter(action_state='N')).distinct()
a_list = exps.filter(
results_set__dmfilestat__in=dmfilestats.filter(action_state='A')).distinct()
logger.info("Experiments (Keep=False) L:%d S:%d N:%d A:%d" %
(l_list.count(), s_list.count(), n_list.count(), a_list.count()), extra=logid)
queue_length = 5
to_select = queue_length - (a_list.count() + n_list.count() + s_list.count())
if to_select > 0:
# Mark to_select number of oldest Experiments, from 'L', to 'S'
promoted = dmfilestats.filter(
result__experiment__id__in=list(l_list[:to_select].values_list('id', flat=True)))
if auto_action_enabled:
promoted.update(action_state='S')
for dmfilestat in promoted:
EventLog.objects.add_entry(
dmfilestat.result, "Signal Processing Input Selected for Deletion", username='******')
# Get updated list of Selected items
selected = dmfilestats.filter(action_state='S')
# Send Selected items to be notified
if selected.count() > 0 and auto_action_enabled:
logger.debug("notify recipient %s" % (recipient), extra=logid)
if notify([dmfilestat.result.resultsName for dmfilestat in selected], recipient):
selected.update(action_state='N')
for dmfilestat in selected:
EventLog.objects.add_entry(
dmfilestat.result, "Notification for Deletion Sent", username='******')
try:
actiondmfilestat = getfirstnotpreserved(
dmfilestats.filter(action_state='A'), dmactions.DELETE, threshdate)
logger.info("Picked: %s" % (actiondmfilestat.result.resultsName), extra=logid)
except DMExceptions.NoDMFileStat:
logger.info("No filesets to delete on this device", extra=logid)
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id), extra=logid)
except:
logger.error(traceback.format_exc(), extra=logid)
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id), extra=logid)
else:
try:
actiondmfilestat = getfirstnotpreserved(dmfilestats, dmactions.DELETE, threshdate)
logger.info("Picked: %s" % (actiondmfilestat.result.resultsName), extra=logid)
except DMExceptions.NoDMFileStat:
logger.info("No filesets to delete on this device", extra=logid)
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id), extra=logid)
except:
logger.error(traceback.format_exc(), extra=logid)
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" %
(os.getpid(), lock_id), extra=logid)
# Bail out if disabled
if auto_action_enabled != True:
logger.info("Data management auto-action is disabled.", extra=logid)
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed %s" % (os.getpid(), lock_id), extra=logid)
return
if actiondmfilestat is not None:
applock.update(actiondmfilestat.result.resultsName)
delete_action('dm_agent', user_comment, actiondmfilestat, lock_id,
confirmed=getattr(actiondmfilestat, 'allow_delete', False))
else:
logger.error("Unknown or unhandled action: %s" % dmfileset['auto_action'], extra=logid)
applock.unlock()
except (DMExceptions.FilePermission,
DMExceptions.InsufficientDiskSpace,
DMExceptions.MediaNotSet,
DMExceptions.MediaNotAvailable,
DMExceptions.FilesInUse) as e:
applock.unlock()
message = Message.objects.filter(tags__contains=e.tag)
if not message:
Message.error(e.message, tags=e.tag)
# TODO: TS-6525: This logentry will repeat for an Archive action every 30 seconds until the cause of the exception is fixed.
# at least, while the message banner is raised, suppress additional Log Entries.
EventLog.objects.add_entry(actiondmfilestat.result, "%s - %s" %
(dmfileset['type'], e.message), username='******')
except DMExceptions.SrcDirDoesNotExist as e:
applock.unlock()
msg = "Src Dir not found: %s. Setting action_state to Deleted" % e.message
EventLog.objects.add_entry(actiondmfilestat.result, msg, username='******')
actiondmfilestat.setactionstate('DD')
logger.info(msg, extra=logid)
except Exception as inst:
applock.unlock()
msg = ''
if actiondmfilestat:
msg = "Auto-action error on %s " % actiondmfilestat.result.resultsName
msg += " Error: %s" % str(inst)
logger.error("%s - %s" % (dmfileset['type'], msg), extra=logid)
logger.error(traceback.format_exc(), extra=logid)
if actiondmfilestat:
EventLog.objects.add_entry(actiondmfilestat.result, "%s - %s" %
(dmfileset['type'], msg), username='******')
return
def manage_manual_action():
logid = {'logid': "%s" % ('manual_action')}
logger.debug("manage_manual_action", extra=logid)
def getfirstmanualaction():
manual_selects = DMFileStat.objects.filter(action_state__in=['SA', 'SE', 'SD']).order_by('created')
for item in manual_selects:
try:
dmactions.action_validation(item, item.dmfileset.auto_action)
return item
except(DMExceptions.FilesInUse, DMExceptions.FilesMarkedKeep):
logger.debug("%s Failed action_validation. Try next fileset" %
item.result.resultsName, extra=logid)
except DMExceptions.BaseInputLinked:
# want to allow Basecalling Input delete if all results are expired
related_objs = DMFileStat.objects.filter(
result__experiment=item.result.experiment, dmfileset__type=dmactions_types.BASE)
if related_objs.count() == related_objs.filter(created__lt=threshdate).count():
item.allow_delete = True
return item
else:
logger.debug("%s Failed action_validation. Try next fileset" %
item.result.resultsName, extra=logid)
except:
logger.error(traceback.format_exc(), extra=logid)
raise DMExceptions.NoDMFileStat("NONE FOUND")
try:
# Create lock file to prevent more than one celery task for manual action (export or archive)
lock_id = 'manual_action_lock_id'
logid = {'logid': "%s" % (lock_id)}
applock = TaskLock(lock_id)
if not applock.lock():
logger.info("Currently processing: %s(%s)" % (lock_id, applock.get()), extra=logid)
return
logger.debug("lock file created: %s(%s)" % (lock_id, applock.get()), extra=logid)
except Exception as e:
logger.error(traceback.format_exc(), extra=logid)
try:
#
# Check for manually selected Archive and Export actions - action_state == 'SA' or 'SE' or 'SD'
# Manual Delete actions do not get processed here. Only suspended Delete actions get processed here.
# These jobs should execute even when auto action is disabled.
# Note: manual actions will not be executed in the order they are selected, but by age.
#
actiondmfilestat = getfirstmanualaction()
user = actiondmfilestat.user_comment.get('user', 'dm_agent')
user_comment = actiondmfilestat.user_comment.get('user_comment', 'Manual Action')
logger.info("Picked: %s" % (actiondmfilestat.result.resultsName), extra=logid)
dmfileset = actiondmfilestat.dmfileset
applock.update(actiondmfilestat.result.resultsName)
if actiondmfilestat.action_state == 'SA':
logger.info("Manual Archive Action: %s from %s" %
(dmfileset.type, actiondmfilestat.result.resultsName), extra=logid)
archive_action(user, user_comment, actiondmfilestat, lock_id, msg_banner=True)
elif actiondmfilestat.action_state == 'SE':
logger.info("Manual Export Action: %s from %s" %
(dmfileset.type, actiondmfilestat.result.resultsName), extra=logid)
export_action(user, user_comment, actiondmfilestat, lock_id, msg_banner=True)
elif actiondmfilestat.action_state == 'SD':
logger.info("Delete Action: %s from %s" %
(dmfileset.type, actiondmfilestat.result.resultsName), extra=logid)
delete_action(user, "Continuing delete action after being suspended",
actiondmfilestat, lock_id, msg_banner=True)
else:
logger.warn("Dev Error: we don't handle this '%s' here" %
actiondmfilestat.action_state, extra=logid)
except DMExceptions.NoDMFileStat:
applock.unlock()
logger.debug("Worker PID %d lock_id destroyed on exit %s" % (os.getpid(), lock_id), extra=logid)
# NOTE: all these exceptions are also handled in manage_data() below. beaucoup de duplication de code
except (DMExceptions.FilePermission,
DMExceptions.InsufficientDiskSpace,
DMExceptions.MediaNotSet,
DMExceptions.MediaNotAvailable,
DMExceptions.FilesInUse) as e:
applock.unlock()
message = Message.objects.filter(tags__contains=e.tag)
if not message:
Message.error(e.message, tags=e.tag)
# TODO: TS-6525: This logentry will repeat for an Archive action every 30 seconds until the cause of the exception is fixed.
# at least, while the message banner is raised, suppress additional Log Entries.
dmactions.add_eventlog(actiondmfilestat, "%s - %s" %
(dmfileset.type, e.message), username='******')
# State needs to be Error since we do not know previous state (ie, do NOT
# set to Local, it might get deleted automatically)
actiondmfilestat.setactionstate('E')
except DMExceptions.SrcDirDoesNotExist as e:
applock.unlock()
msg = "Src Dir not found: %s. Setting action_state to Deleted" % e.message
EventLog.objects.add_entry(actiondmfilestat.result, msg, username='******')
actiondmfilestat.setactionstate('DD')
logger.info(msg, extra=logid)
except Exception as e:
applock.unlock()
msg = "action error on %s " % actiondmfilestat.result.resultsName
msg += " Error: %s" % str(e)
logger.error("%s - %s" % (dmfileset.type, msg), extra=logid)
logger.error(traceback.format_exc(), extra=logid)
dmactions.add_eventlog(actiondmfilestat, "%s - %s" % (dmfileset.type, msg), username='******')
# State needs to be Error since we do not know previous state (ie, do NOT
# set to Local, it might get deleted automatically)
actiondmfilestat.setactionstate('E')
return
#!/usr/bin python
# Copyright (C) 2011 Ion Torrent Systems, Inc. All Rights Reserved
import argparse
from djangoinit import *
from iondb.rundb.models import Message
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Post a site message to the Torrent browser.")
parser.add_argument("message", type=str, help="The message string to be displayed.")
parser.add_argument("-d", "--duplicate", action="store_true",
help="Create a duplicate post if necessary.")
args = parser.parse_args()
msg = Message.objects.filter(body=args.message).count()
if not msg or args.duplicate:
Message.info(args.message)
print("Posted message")
else:
print("Didn't post duplicate message")
