def config_publishers(request, ctx):
globalconfig = GlobalConfig.get()
# Rescan Publishers
publishers.purge_publishers()
publishers.search_for_publishers(globalconfig)
pubs = Publisher.objects.all().order_by('name')
ctx.update({"publishers": pubs})
def send_deep_laser_metrics(event_type, event_data, schema=1):
if schema == 1:
payload = {
"request": "senddata",
"appName": "Torrent Suite",
"appVersion": TS_version,
"datatype": "metric",
"eventId": event_type,
"parameters": event_data,
}
payload["parameters"]["serverSerial"] = get_servicetag()
payload["parameters"]["siteName"] = GlobalConfig.get().site_name
return send_deep_laser_iot_request(payload)
elif schema == 2:
raise DeprecationWarning
# payload = {
# "request": "senddata",
# "application": "Torrent Suite",
# "applicationVersion": TS_version,
# "timestamp": int(time.time()),
# "datatype": "metric",
# "eventId": event_type,
# "eventParameters": event_data,
# }
# payload["eventParameters"]["serverSerial"] = get_servicetag()
# payload["eventParameters"]["siteName"] = GlobalConfig.get().site_name
# return send_deep_laser_iot_request(payload)
else:
raise ValueError("send_deep_laser_metrics schema must be 1 or 2")
def monitor(request):
"""This is a the main entry point to the Monitor tab."""
pageSize = GlobalConfig.get().records_to_display
resource = MonitorResultResource()
objects = resource.get_object_list(request)
paginator = resource._meta.paginator_class(
request.GET,
objects,
resource_uri=resource.get_resource_uri(),
limit=pageSize,
max_limit=resource._meta.max_limit,
collection_name=resource._meta.collection_name,
)
to_be_serialized = paginator.page()
to_be_serialized["objects"] = [
resource.full_dehydrate(resource.build_bundle(obj=obj,
request=request))
for obj in to_be_serialized["objects"]
]
serialized_exps = resource.serialize(None, to_be_serialized,
"application/json")
context = {"initial_runs": serialized_exps, "pageSize": pageSize}
return render_to_response("rundb/monitor/monitor.html",
context,
context_instance=RequestContext(request))
def monitor(request):
"""This is a the main entry point to the Monitor tab."""
pageSize = GlobalConfig.get().records_to_display
resource = MonitorResultResource()
objects = resource.get_object_list(request)
paginator = resource._meta.paginator_class(
request.GET,
objects,
resource_uri=resource.get_resource_uri(),
limit=pageSize,
max_limit=resource._meta.max_limit,
collection_name=resource._meta.collection_name
)
to_be_serialized = paginator.page()
to_be_serialized['objects'] = [
resource.full_dehydrate(resource.build_bundle(obj=obj, request=request))
for obj in to_be_serialized['objects']
]
serialized_exps = resource.serialize(None, to_be_serialized, 'application/json')
context = {
'initial_runs': serialized_exps,
'pageSize' : pageSize
}
return render_to_response("rundb/monitor/monitor.html", context,
context_instance=RequestContext(request))
def configure_configure(request):
ctx = RequestContext(request, {})
emails = EmailAddress.objects.all().order_by('pk')
enable_nightly = GlobalConfig.get().enable_nightly_email
ctx.update({"email": emails, "enable_nightly": enable_nightly})
config_contacts(request, ctx)
config_site_name(request, ctx)
return render_to_response("rundb/configure/configure.html", context_instance=ctx)
def config_site_name(request, context):
"""The site name will be automatically loaded on the page, so all we have
to do here is check whether we should update it, and if so, do so.
"""
if request.method == "POST" and "site_name" in request.POST:
config = GlobalConfig.get()
config.site_name = request.POST["site_name"]
config.save()
context.update({"base_site_name": request.POST["site_name"]})
def news(request):
profile = request.user.userprofile
ctx = {
"articles": list(NewsPost.objects.all().order_by("-updated")),
"last_read": profile.last_read_news_post,
"is_updating": GlobalConfig.get().check_news_posts,
}
profile.last_read_news_post = timezone.now()
profile.save()
return render(request, "rundb/home/news.html", ctx)
def news(request):
profile = request.user.userprofile
ctx = {
"articles": list(NewsPost.objects.all().order_by('-updated')),
"last_read": profile.last_read_news_post,
"is_updating": GlobalConfig.get().check_news_posts,
}
profile.last_read_news_post = timezone.now()
profile.save()
return render_to_response("rundb/extra/news.html", ctx, context_instance=RequestContext(request))
def news(request):
profile = request.user.userprofile
ctx = {
"articles": list(NewsPost.objects.all().order_by('-updated')),
"last_read": profile.last_read_news_post,
"is_updating": GlobalConfig.get().check_news_posts,
}
profile.last_read_news_post = timezone.now()
profile.save()
return render(request, "rundb/home/news.html", ctx)
def fileserver_space_check():
"""For each file server, compare disk usage to backup threshold value.
If disk usage exceeds threshold, launch celery task to delete/archive
raw data directories.
python -c "from iondb.bin import djangoinit; from iondb.rundb.data import data_management as dm; dm.fileserver_space_check()"
"""
logid = {"logid": "%s" % ("DM")}
try:
starttime = time.time()
# Get GlobalConfig object in order to access auto-acknowledge bit
global_config = GlobalConfig.get()
auto_acknowledge = global_config.auto_archive_ack
auto_action_enabled = global_config.auto_archive_enable
category_list = dm_utils.dm_category_list()
# -------------------------------------------------------------
# Action loop - for each category, launch action per deviceid
# -------------------------------------------------------------
if not auto_action_enabled:
logger.info("Data management auto-action is disabled.",
extra=logid)
# update any DMFileStats that are in use by active analysis jobs
try:
update_files_in_use()
except Exception:
logger.error("Unable to update active DMFileStats", extra=logid)
logger.error(traceback.format_exc(), extra=logid)
# Checks for manual export and archive requests.
manage_manual_action.delay()
for category, dict in category_list.items():
for deviceid in dict["devlist"]:
pathlist = [
item["path"] for item in dict["partitions"]
if item["devid"] == deviceid
]
manage_data.delay(
deviceid,
dict["dmfileset"],
pathlist,
auto_acknowledge,
auto_action_enabled,
)
endtime = time.time()
logger.info("Disk Check: %f s" % (endtime - starttime), extra=logid)
except SoftTimeLimitExceeded:
logger.error("fileserver_space_check exceeded execution time limit",
extra=logid)
return
def send_email(recipient, subject_line, text, html=None):
"""sends an email to recipients"""
import socket
from django.core import mail
from iondb.rundb.models import GlobalConfig
if not recipient:
logger.warning("No email recipient for %s" % subject_line)
return False
else:
recipient = recipient.replace(",", " ").replace(";", " ").split()
# Needed to send email
settings.EMAIL_HOST = "localhost"
settings.EMAIL_PORT = 25
settings.EMAIL_USE_TLS = False
site_name = GlobalConfig.get().site_name or "Torrent Server"
hname = socket.getfqdn()
message = "From: %s (%s)\n\n" % (site_name, hname)
message += text
message += "\n"
if html:
html_message = "From: %s (<a href=%s>%s</a>)<br>" % (site_name, hname, hname)
html_message += html
html_message += "<br>"
else:
html_message = ""
reply_to = "*****@*****.**"
# Send the email
try:
if html_message:
sendthis = mail.EmailMultiAlternatives(
subject_line, message, reply_to, recipient
)
sendthis.attach_alternative(html_message, "text/html")
sendthis.send()
else:
mail.send_mail(subject_line, message, reply_to, recipient)
except:
logger.error(traceback.format_exc())
return False
else:
logger.info("%s email sent to %s" % (subject_line, recipient))
return True
def notify(name_list, recipient):
'''sends an email with list of experiments slated for removal'''
# Check for blank email
# TODO: check for valid email address
if recipient is None or recipient == "":
return False
# Needed to send email
settings.EMAIL_HOST = 'localhost'
settings.EMAIL_PORT = 25
settings.EMAIL_USE_TLS = False
try:
site_name = GlobalConfig.get().site_name
except:
site_name = "Torrent Server"
hname = socket.gethostname()
subject_line = 'Torrent Server Data Management Action Request'
reply_to = '*****@*****.**'
message = 'From: %s (%s)\n' % (site_name, hname)
message += '\n'
message += 'Results drive capacity threshold has been reached.\n'
message += 'Signal Processing files have been identified for removal.\n'
message += 'Please go to Services Page and acknowledge so that removal can proceed.\n'
message += 'Removal will not occur without this acknowledgement.\n'
message += '\n'
message += 'The following Reports have Signal Processing files selected for Deletion:'
message += "\n"
count = 0
for e in name_list:
message += "- %s\n" % e
count += 1
# Send the email only if there are runs that have not triggered a notification
if count > 0:
try:
mail.send_mail(subject_line, message, reply_to, recipient)
except:
logger.warning(traceback.format_exc(), extra=logid)
return False
else:
logger.info("Notification email sent for user acknowledgement",
extra=logid)
return True
def send_email(recipient, subject_line, text, html=None):
'''sends an email to recipients'''
import socket
from django.core import mail
from iondb.rundb.models import GlobalConfig
if not recipient:
logger.warning("No email recipient for %s" % subject_line)
return False
else:
recipient = recipient.replace(',', ' ').replace(';', ' ').split()
#Needed to send email
settings.EMAIL_HOST = 'localhost'
settings.EMAIL_PORT = 25
settings.EMAIL_USE_TLS = False
site_name = GlobalConfig.get().site_name or 'Torrent Server'
hname = socket.getfqdn()
message = 'From: %s (%s)\n\n' % (site_name, hname)
message += text
message += '\n'
if html:
html_message = 'From: %s (<a href=%s>%s</a>)<br>' % (site_name, hname, hname)
html_message += html
html_message += '<br>'
else:
html_message = ''
reply_to = '*****@*****.**'
# Send the email
try:
if html_message:
sendthis = mail.EmailMultiAlternatives(subject_line, message, reply_to, recipient)
sendthis.attach_alternative(html_message, "text/html")
sendthis.send()
else:
mail.send_mail(subject_line, message, reply_to, recipient)
except:
logger.error(traceback.format_exc())
return False
else:
logger.info("%s email sent to %s" % (subject_line, recipient))
return True
def notify(name_list, recipient):
'''sends an email with list of experiments slated for removal'''
# Check for blank email
# TODO: check for valid email address
if recipient is None or recipient == "":
return False
# Needed to send email
settings.EMAIL_HOST = 'localhost'
settings.EMAIL_PORT = 25
settings.EMAIL_USE_TLS = False
try:
site_name = GlobalConfig.get().site_name
except:
site_name = "Torrent Server"
hname = socket.gethostname()
subject_line = 'Torrent Server Data Management Action Request'
reply_to = '*****@*****.**'
message = 'From: %s (%s)\n' % (site_name, hname)
message += '\n'
message += 'Results drive capacity threshold has been reached.\n'
message += 'Signal Processing files have been identified for removal.\n'
message += 'Please go to Services Page and acknowledge so that removal can proceed.\n'
message += 'Removal will not occur without this acknowledgement.\n'
message += '\n'
message += 'The following Reports have Signal Processing files selected for Deletion:'
message += "\n"
count = 0
for e in name_list:
message += "- %s\n" % e
count += 1
# Send the email only if there are runs that have not triggered a notification
if count > 0:
try:
mail.send_mail(subject_line, message, reply_to, recipient)
except:
logger.warning(traceback.format_exc(), extra=logid)
return False
else:
logger.info("Notification email sent for user acknowledgement", extra=logid)
return True
def showpage(request):
"""Main Chips Files Display"""
site_name = GlobalConfig.get().site_name
# search all File Servers for a "Chips" directory
fileservers = FileServer.objects.all()
files = {}
locList = []
for server in fileservers:
directory = os.path.join(server.filesPrefix, "Chips")
if os.path.isdir(directory):
files[server.name] = []
listoffiles = os.listdir(directory)
listoffiles.sort()
listoffiles.reverse()
for filename in listoffiles:
if fnmatch.fnmatch(filename, "*AutoPH*.bz2"):
# instName = string.split(filename, '_')[0]
instName = filename.split("_")[0]
if not [instName, server.name] in locList:
locList.append([instName, server.name])
if fnmatch.fnmatch(filename, "*AutoPHFail*"):
passVar = "F"
elif fnmatch.fnmatch(filename, "*AutoPHPass*"):
passVar = "T"
files[server.name].append([
filename.split(".")[0],
instName,
os.path.join(directory, filename),
passVar,
])
protonDiags = findProtonDiags(fileservers)
ctxd = {
"error_state": 0,
"locations_list": locList,
"base_site_name": site_name,
"files": files,
"protonDiags": protonDiags,
}
ctx = RequestContext(request, ctxd)
return render_to_response("rundb/configure/ion_chips.html",
context_instance=ctx)
def fileserver_space_check():
'''For each file server, compare disk usage to backup threshold value.
If disk usage exceeds threshold, launch celery task to delete/archive
raw data directories.
python -c "from iondb.bin import djangoinit; from iondb.rundb.data import data_management as dm; dm.fileserver_space_check()"
'''
logid = {'logid': "%s" % ('DM')}
try:
starttime = time.time()
# Get GlobalConfig object in order to access auto-acknowledge bit
global_config = GlobalConfig.get()
auto_acknowledge = global_config.auto_archive_ack
auto_action_enabled = global_config.auto_archive_enable
category_list = dm_utils.dm_category_list()
#-------------------------------------------------------------
# Action loop - for each category, launch action per deviceid
#-------------------------------------------------------------
if not auto_action_enabled:
logger.info("Data management auto-action is disabled.", extra=logid)
# update any DMFileStats that are in use by active analysis jobs
try:
update_files_in_use()
except:
logger.error('Unable to update active DMFileStats', extra=logid)
logger.error(traceback.format_exc(), extra=logid)
# Checks for manual export and archive requests.
manage_manual_action.delay()
for category, dict in category_list.iteritems():
for deviceid in dict['devlist']:
pathlist = [item['path'] for item in dict['partitions'] if item['devid'] == deviceid]
manage_data.delay(
deviceid, dict['dmfileset'], pathlist, auto_acknowledge, auto_action_enabled)
endtime = time.time()
logger.info("Disk Check: %f s" % (endtime - starttime), extra=logid)
except SoftTimeLimitExceeded:
logger.error("fileserver_space_check exceeded execution time limit", extra=logid)
return
def showpage(request):
'''Main Chips Files Display'''
site_name = GlobalConfig.get().site_name
# search all File Servers for a "Chips" directory
fileservers = FileServer.objects.all()
files = {}
locList = []
for server in fileservers:
directory = os.path.join(server.filesPrefix, 'Chips')
if os.path.isdir(directory):
files[server.name] = []
listoffiles = os.listdir(directory)
listoffiles.sort()
listoffiles.reverse()
for filename in listoffiles:
if fnmatch.fnmatch(filename, "*AutoPH*.bz2"):
#instName = string.split(filename, '_')[0]
instName = filename.split('_')[0]
if not [instName, server.name] in locList:
locList.append([instName, server.name])
if fnmatch.fnmatch(filename, "*AutoPHFail*"):
passVar = 'F'
elif fnmatch.fnmatch(filename, "*AutoPHPass*"):
passVar = 'T'
files[server.name].append([filename.split('.')[0], instName, os.path.join(directory, filename), passVar])
protonDiags = findProtonDiags(fileservers)
ctxd = {
"error_state": 0,
"locations_list": locList,
"base_site_name": site_name,
"files": files,
"protonDiags": protonDiags,
}
ctx = RequestContext(request, ctxd)
return render_to_response("rundb/configure/ion_chips.html", context_instance=ctx)
def setUp(self):
self.tempdir = tempfile.mkdtemp()
self.assertEqual(1, len(GlobalConfig.objects.all().order_by('pk')))
self.gc = GlobalConfig.get()
def setUp(self):
self.tempdir = tempfile.mkdtemp()
self.assertEqual(1, len(GlobalConfig.objects.all().order_by("pk")))
self.gc = GlobalConfig.get()
def dashboard_fragments(request, skip_runs=False):
""" Returns the dashboard sections as html in a json object"""
time_span = request.GET.get("time_span", "24hours")
now = datetime.datetime.now(pytz.UTC)
DASHBOARD_TIME_SPANS = {
"hour": now - datetime.timedelta(hours=1),
"today": now.replace(hour=0, minute=0, second=0, microsecond=0),
"24hours": now - datetime.timedelta(hours=24),
"7days": now - datetime.timedelta(days=7),
# Used for testing only. Do not expose to the UI.
"__all__": datetime.datetime(year=1971, month=1, day=1),
}
if time_span not in DASHBOARD_TIME_SPANS:
raise Http404("Time span %s not available!" % time_span)
# runs section
if skip_runs:
runs_context = {
# Runs Section
"runs": {
"time_span": time_span,
"stages": DASHBOARD_STAGES,
"runs": [],
"error": "",
}
}
else:
try:
runs = get_runs_list(DASHBOARD_TIME_SPANS[time_span])
runs_error = None
except Exception as err:
runs = []
runs_error = str(err)
runs_context = {
# Runs Section
"runs": {
"time_span": time_span,
"stages": DASHBOARD_STAGES,
"runs": runs,
"error": runs_error,
}
}
# software update
update_status = GlobalConfig.get().ts_update_status
# services
services_down = []
for process, state in process_set():
if not state:
services_down.append(process)
show_cluster = False
nodes_down = []
if Cruncher.objects.count() > 0:
show_cluster = True
nodes_down = Cruncher.objects.exclude(state="G").values_list("name",
flat=True)
# storage status
storage = get_storage_status()
# data management
disk_usage = get_disk_usage()
dm_active_jobs = DMFileStat.objects.filter(
action_state__in=["AG", "DG", "EG", "SA", "SE", "SD", "IG"
]).values_list("action_state", flat=True)
dm_errors = DMFileStat.objects.filter(action_state="E").count()
# instruments
rigs = Rig.objects.exclude(host_address="")
num_rigs = len(rigs)
if num_rigs > 1:
with ManagedPool(processes=min(num_rigs, 50)) as pool:
instruments = pool.map(get_instrument_info, rigs)
else:
instruments = [get_instrument_info(rig) for rig in rigs]
instr_connected = sum(
[instr["status"] == CONNECTED for instr in instruments])
instr_offline = sum([instr["status"] == OFFLINE for instr in instruments])
instr_alarm = sum([instr["status"] == ALARM for instr in instruments])
summary_context = {
# Summary Section
"summary": {
"ts_version": TS_version,
"update_status": update_status,
"instruments": {
"connected": instr_connected,
"offline": instr_offline,
"alerts": instr_alarm,
},
"services": {
"url": reverse("configure_services"),
"number_services_down": len(services_down),
"services_down": services_down,
"show_cluster": True if show_cluster else False,
"number_nodes_down": len(nodes_down) if show_cluster else "",
"show_nas": storage["show_nas"],
"nas_status": storage.get("nas_status", ""),
"show_raid": storage["show_raid"],
"raid_status": storage.get("raid_status", ""),
},
"data_management": {
"url":
reverse("datamanagement"),
"disk_usage":
disk_usage,
"show_path":
len(disk_usage) > 1,
"dm_jobs": [
("archive in progress",
sum([s == "AG" for s in dm_active_jobs])),
("export in progress",
sum([s == "EG" for s in dm_active_jobs])),
("delete in progress",
sum([s == "DG" for s in dm_active_jobs])),
("import in progress",
sum([s == "IG" for s in dm_active_jobs])),
("archive pending",
sum([s == "SA" for s in dm_active_jobs])),
("export pending", sum([s == "SE"
for s in dm_active_jobs])),
("delete pending", sum([s == "SD"
for s in dm_active_jobs])),
],
"dm_errors":
dm_errors,
},
}
}
instruments_context = {
"instruments":
sorted(instruments, key=lambda x: (x["status"], x["name"].lower()))
}
return HttpResponse(
json.dumps({
"summary":
render_to_string("rundb/home/fragments/summary.html",
summary_context),
"runs":
render_to_string("rundb/home/fragments/runs.html", runs_context),
"instruments":
render_to_string("rundb/home/fragments/instruments.html",
instruments_context),
}),
content_type="application/json",
)
def __init__(self, sh_type):
super(KitsStepData, self).__init__(sh_type)
self.resourcePath = 'rundb/plan/page_plan/page_plan_kits.html'
#20130827-test
##self._dependsOn.append(StepNames.IONREPORTER)
self._dependsOn.append(StepNames.APPLICATION)
self._dependsOn.append(StepNames.BARCODE_BY_SAMPLE)
self.savedFields[KitsFieldNames.SAMPLE_PREPARATION_KIT] = None
self.prepopulatedFields[KitsFieldNames.SAMPLE_PREP_KITS] = KitInfo.objects.filter(kitType='SamplePrepKit', isActive=True).order_by('description')
self.savedFields[KitsFieldNames.LIBRARY_KIT_NAME] = None
self.prepopulatedFields[KitsFieldNames.LIB_KITS] = KitInfo.objects.filter(kitType='LibraryKit', isActive=True).order_by("description")
self.savedFields[KitsFieldNames.LIBRARY_KEY] = None
self.prepopulatedFields[KitsFieldNames.FORWARD_LIB_KEYS] = LibraryKey.objects.filter(direction='Forward', runMode='single').order_by('-isDefault', 'name')
self.savedFields[KitsFieldNames.LIBRARY_KEY] = self.prepopulatedFields[KitsFieldNames.FORWARD_LIB_KEYS][0].sequence
self.savedFields[KitsFieldNames.TF_KEY] = GlobalConfig.get().default_test_fragment_key
self.savedFields[KitsFieldNames.FORWARD_3_PRIME_ADAPTER] = None
self.prepopulatedFields[KitsFieldNames.FORWARD_3_ADAPTERS] = ThreePrimeadapter.objects.filter(direction='Forward', runMode='single').order_by('-isDefault', 'chemistryType', 'name')
self.savedFields[KitsFieldNames.FORWARD_3_PRIME_ADAPTER] = self.prepopulatedFields[KitsFieldNames.FORWARD_3_ADAPTERS][0].sequence
self.savedFields[KitsFieldNames.AVALANCHE_FORWARD_3_PRIME_ADAPTER] = None
self.prepopulatedFields[KitsFieldNames.AVALANCHE_FORWARD_3_PRIME_ADAPTERS] = ThreePrimeadapter.objects.filter(direction='Forward', runMode='single', chemistryType = 'avalanche').order_by('-isDefault', 'name')
self.savedFields[KitsFieldNames.AVALANCHE_FORWARD_3_PRIME_ADAPTER] = self.prepopulatedFields[KitsFieldNames.AVALANCHE_FORWARD_3_PRIME_ADAPTERS][0].sequence
self.savedFields[KitsFieldNames.TEMPLATE_KIT_NAME] = None
##no longer default to OneTouch
#self.savedFields[KitsFieldNames.TEMPLATE_KIT_TYPE] = KitsFieldNames.ONE_TOUCH
self.savedFields[KitsFieldNames.TEMPLATE_KIT_TYPE] = None
oneTouchDict = {
KitsFieldNames.KIT_VALUES : KitInfo.objects.filter(kitType__in=['TemplatingKit', 'AvalancheTemplateKit'], isActive=True).order_by("description"),
KitsFieldNames.APPLICATION_DEFAULT : None
}
ionChefDict = {
KitsFieldNames.KIT_VALUES : KitInfo.objects.filter(kitType='IonChefPrepKit', isActive=True).order_by("description"),
KitsFieldNames.APPLICATION_DEFAULT : None
}
oneTouchAvalancheDict = {
KitsFieldNames.KIT_VALUES : KitInfo.objects.filter(kitType__in=['AvalancheTemplateKit'], isActive=True).order_by("description"),
KitsFieldNames.APPLICATION_DEFAULT : None
}
self.prepopulatedFields[KitsFieldNames.TEMPLATE_KIT_TYPES] = {
KitsFieldNames.ONE_TOUCH : oneTouchDict,
KitsFieldNames.ION_CHEF : ionChefDict,
KitsFieldNames.ONE_TOUCH_AVALANCHE : oneTouchAvalancheDict
}
self.savedFields[KitsFieldNames.SEQUENCE_KIT_NAME] = None
self.prepopulatedFields[KitsFieldNames.SEQ_KITS] = KitInfo.objects.filter(kitType='SequencingKit', isActive=True).order_by("description")
self.savedFields[KitsFieldNames.TEMPLATE_KIT_NAME] = None
self.prepopulatedFields[KitsFieldNames.TEMPLATE_KITS] = KitInfo.objects.filter(kitType__in=['TemplatingKit', 'AvalancheTemplateKit'], isActive=True).order_by("description")
self.prepopulatedFields[KitsFieldNames.ALL_TEMPLATE_KITS] = KitInfo.objects.filter(kitType__in=['TemplatingKit', 'AvalancheTemplateKit', 'IonChefPrepKit'], isActive=True).order_by("description")
self.savedFields[KitsFieldNames.CONTROL_SEQUENCE] = None
self.prepopulatedFields[KitsFieldNames.CONTROL_SEQ_KITS] = KitInfo.objects.filter(kitType='ControlSequenceKit', isActive=True).order_by("description")
self.savedFields[KitsFieldNames.CHIP_TYPE] = None
self.prepopulatedFields[KitsFieldNames.CHIP_TYPES] = list(Chip.objects.filter(isActive=True).order_by('description', 'name').distinct('description'))
self.savedFields[KitsFieldNames.BARCODE_ID] = None
self.prepopulatedFields[KitsFieldNames.BARCODES] = list(dnaBarcode.objects.values('name').distinct().order_by('name'))
gc = GlobalConfig.get()
self.savedFields[KitsFieldNames.IS_DUPLICATED_READS] = gc.mark_duplicates
self.savedFields[KitsFieldNames.BASE_RECALIBRATE]= gc.base_recalibration_mode
self.prepopulatedFields[KitsFieldNames.BASE_RECALIBRATION_MODES] = OrderedDict()
self.prepopulatedFields[KitsFieldNames.BASE_RECALIBRATION_MODES]["standard_recal"] = "Default Calibration"
self.prepopulatedFields[KitsFieldNames.BASE_RECALIBRATION_MODES]["panel_recal"] = "Enable Calibration Standard"
self.prepopulatedFields[KitsFieldNames.BASE_RECALIBRATION_MODES]["no_recal"] = "No Calibration"
self.savedFields[KitsFieldNames.REALIGN] = gc.realign
self.savedFields[KitsFieldNames.FLOWS] = 0
self.savedFields[KitsFieldNames.LIBRARY_READ_LENGTH] = 0
self.prepopulatedFields[KitsFieldNames.IS_BARCODE_KIT_SELECTION_REQUIRED] = False
self.prepopulatedFields[KitsFieldNames.TEMPLATING_SIZE_CHOICES] = ["200", "400"]
self.savedFields[KitsFieldNames.TEMPLATING_SIZE] = ""
self.sh_type = sh_type
def isOCP_enabled():
return GlobalConfig.get().enable_compendia_OCP
def fileserver_space_check():
'''For each file server, compare disk usage to backup threshold value.
If disk usage exceeds threshold, launch celery task to delete/archive
raw data directories.
'''
logger = get_task_logger('data_management')
# Get GlobalConfig object in order to access auto-acknowledge bit
gc = GlobalConfig.get()
auto_acknowledge = gc.auto_archive_ack
auto_action_enabled = gc.auto_archive_enable
# Get list of File Server objects
file_servers = FileServer.objects.all().order_by('pk').values()
# Get list of Report Storage objects
report_storages = ReportStorage.objects.all().order_by('pk').values()
# dict of fileset cateogries each with list of partition ids that can be acted upon.
category_list = {}
#-------------------------------------------------
# DELETE action only happens if threshold reached
#-------------------------------------------------
for dmfileset in DMFileSet.objects.filter(version=settings.RELVERSION).filter(auto_action='DEL').values():
cat_name = slugify(dmfileset['type'])
category_list[cat_name] = {
'dmfileset':dmfileset,
'devlist':[],
'partitions':[],
}
for partition in _partitions(file_servers,report_storages):
if partition['diskusage'] >= dmfileset['auto_trigger_usage']:
logger.info("%s %s %.2f%% exceeds %s %.0f%%" % (
hex(partition['devid']),
partition['path'],
partition['diskusage'],
dmfileset['type'],
dmfileset['auto_trigger_usage']))
category_list[cat_name]['devlist'].append(partition['devid'])
category_list[cat_name]['partitions'].append(partition)
else:
logger.info("%s %s %.2f%% below %s %.0f%%" % (
hex(partition['devid']),
partition['path'],
partition['diskusage'],
dmfileset['type'],
dmfileset['auto_trigger_usage']))
# uniquify the deviceid list
category_list[cat_name]['devlist'] = list(set(category_list[cat_name]['devlist']))
#-------------------------------------------------------------------------------
#ARCHIVE action happens as soon as grace period has expired (no threshold check)
#-------------------------------------------------------------------------------
for dmfileset in DMFileSet.objects.filter(version=settings.RELVERSION).filter(auto_action='ARC').values():
cat_name = slugify(dmfileset['type'])
category_list[cat_name] = {
'dmfileset':dmfileset,
'devlist':[],
'partitions':[],
}
for partition in _partitions(file_servers,report_storages):
logger.debug("%s %s" %( partition['path'],hex(partition['devid'])))
category_list[cat_name]['devlist'].append(partition['devid'])
category_list[cat_name]['partitions'].append(partition)
# uniquify the deviceid list
category_list[cat_name]['devlist'] = list(set(category_list[cat_name]['devlist']))
#-------------------------------------------------------------
# Action loop - for each category, launch action per deviceid
#-------------------------------------------------------------
if not auto_action_enabled:
logger.info("Data management auto-action is disabled.")
# update any DMFileStats that are in use by active analysis jobs
try:
update_files_in_use()
except:
logger.error('Unable to update active DMFileStats')
logger.error(traceback.format_exc())
# Checks for manual export and archive requests.
manage_manual_action.delay()
for category,dict in category_list.iteritems():
for deviceid in dict['devlist']:
pathlist = [item['path'] for item in dict['partitions'] if item['devid'] == deviceid]
manage_data.delay(deviceid, dict['dmfileset'], pathlist, auto_acknowledge, auto_action_enabled)
return
def __init__(self, sh_type):
super(KitsStepData, self).__init__(sh_type)
self.resourcePath = "rundb/plan/page_plan/page_plan_kits.html"
self.prev_step_url = reverse("page_plan_application")
self.next_step_url = reverse("page_plan_plugins")
# 20130827-test
# self._dependsOn.append(StepNames.IONREPORTER)
self._dependsOn.append(StepNames.APPLICATION)
self._dependsOn.append(StepNames.BARCODE_BY_SAMPLE)
self.savedFields[KitsFieldNames.SAMPLE_PREPARATION_KIT] = None
self.prepopulatedFields[
KitsFieldNames.SAMPLE_PREP_KITS] = KitInfo.objects.filter(
kitType="SamplePrepKit", isActive=True).order_by("description")
self.savedFields[KitsFieldNames.LIBRARY_KIT_NAME] = None
self.prepopulatedFields[
KitsFieldNames.LIB_KITS] = KitInfo.objects.filter(
kitType__in=["LibraryKit", "LibraryPrepKit"],
isActive=True).order_by("description")
self.savedFields[KitsFieldNames.LIBRARY_KEY] = None
self.prepopulatedFields[
KitsFieldNames.FORWARD_LIB_KEYS] = LibraryKey.objects.filter(
direction="Forward",
runMode="single").order_by("-isDefault", "name")
self.savedFields[KitsFieldNames.LIBRARY_KEY] = self.prepopulatedFields[
KitsFieldNames.FORWARD_LIB_KEYS][0].sequence
self.savedFields[KitsFieldNames.TF_KEY] = GlobalConfig.get(
).default_test_fragment_key
self.savedFields[KitsFieldNames.FORWARD_3_PRIME_ADAPTER] = None
self.prepopulatedFields[
KitsFieldNames.
FORWARD_3_ADAPTERS] = ThreePrimeadapter.objects.filter(
direction="Forward",
runMode="single").order_by("-isDefault", "chemistryType",
"name")
self.savedFields[
KitsFieldNames.FORWARD_3_PRIME_ADAPTER] = self.prepopulatedFields[
KitsFieldNames.FORWARD_3_ADAPTERS][0].sequence
self.savedFields[KitsFieldNames.FLOW_ORDER] = None
self.prepopulatedFields[
KitsFieldNames.FLOW_ORDERS] = FlowOrder.objects.filter(
isActive=True).order_by("-isDefault", "description")
self.savedFields[KitsFieldNames.FLOW_ORDER] = None
self.savedFields[KitsFieldNames.TEMPLATE_KIT_NAME] = None
# no longer default to OneTouch
# self.savedFields[KitsFieldNames.TEMPLATE_KIT_TYPE] = KitsFieldNames.ONE_TOUCH
self.savedFields[KitsFieldNames.TEMPLATE_KIT_TYPE] = None
oneTouchDict = {
KitsFieldNames.KIT_VALUES:
KitInfo.objects.filter(
kitType__in=["TemplatingKit", "AvalancheTemplateKit"],
isActive=True).exclude(
samplePrep_instrumentType="IA").order_by("description"),
KitsFieldNames.APPLICATION_DEFAULT:
None,
}
isoAmpDict = {
KitsFieldNames.KIT_VALUES:
KitInfo.objects.filter(
kitType__in=["TemplatingKit"],
isActive=True,
samplePrep_instrumentType="IA",
).order_by("description"),
KitsFieldNames.APPLICATION_DEFAULT:
None,
}
ionChefDict = {
KitsFieldNames.KIT_VALUES:
KitInfo.objects.filter(kitType="IonChefPrepKit",
isActive=True).order_by("description"),
KitsFieldNames.APPLICATION_DEFAULT:
None,
}
self.prepopulatedFields[
KitsFieldNames.TEMPLATE_KIT_TYPES] = OrderedDict([
(KitsFieldNames.ONE_TOUCH, oneTouchDict),
(KitsFieldNames.ION_CHEF, ionChefDict),
(KitsFieldNames.ISO_AMP, isoAmpDict),
])
self.ModelsSamplePrepInstrumentToLabelsSamplePrepInstrumentAsDict = {
KitsFieldNames.ONE_TOUCH: SamplePrepInstrument.OT,
KitsFieldNames.ION_CHEF: SamplePrepInstrument.IC,
KitsFieldNames.ISO_AMP: SamplePrepInstrument.IA,
}
self.savedFields[KitsFieldNames.SEQUENCE_KIT_NAME] = None
self.prepopulatedFields[
KitsFieldNames.SEQ_KITS] = KitInfo.objects.filter(
kitType="SequencingKit", isActive=True).order_by("description")
self.savedFields[KitsFieldNames.TEMPLATE_KIT_NAME] = None
self.prepopulatedFields[
KitsFieldNames.TEMPLATE_KITS] = KitInfo.objects.filter(
kitType__in=[
"TemplatingKit", "AvalancheTemplateKit", "IonChefPrepKit"
],
isActive=True,
).order_by("description")
self.savedFields[KitsFieldNames.CONTROL_SEQUENCE] = None
self.prepopulatedFields[
KitsFieldNames.CONTROL_SEQ_KITS] = KitInfo.objects.filter(
kitType="ControlSequenceKit",
isActive=True).order_by("description")
self.savedFields[KitsFieldNames.CHIP_TYPE] = None
self.prepopulatedFields[
KitsFieldNames.
INSTRUMENT_TYPES] = Chip.getInstrumentTypesForActiveChips(
include_undefined=False)
self.prepopulatedFields[KitsFieldNames.CHIP_TYPES] = list(
Chip.objects.filter(isActive=True).order_by(
"description", "name").distinct("description"))
self.savedFields[KitsFieldNames.BARCODE_ID] = None
self.prepopulatedFields[KitsFieldNames.BARCODES] = list(
dnaBarcode.objects.filter(
active=True).values("name").distinct().order_by("name"))
gc = GlobalConfig.get()
self.savedFields[
KitsFieldNames.IS_DUPLICATED_READS] = gc.mark_duplicates
self.savedFields[
KitsFieldNames.BASE_RECALIBRATE] = gc.base_recalibration_mode
self.prepopulatedFields[
KitsFieldNames.BASE_RECALIBRATION_MODES] = OrderedDict()
self.prepopulatedFields[
KitsFieldNames.BASE_RECALIBRATION_MODES]["standard_recal"] = {
"text":
_("workflow.step.kits.base_recalibration_modes.standard_recal"
),
"title":
_("workflow.step.kits.base_recalibration_modes.standard_recal.title"
),
} # "Default Calibration"
self.prepopulatedFields[
KitsFieldNames.BASE_RECALIBRATION_MODES]["panel_recal"] = {
"text":
_("workflow.step.kits.base_recalibration_modes.panel_recal"),
"title":
_("workflow.step.kits.base_recalibration_modes.panel_recal.title"
),
} # "Enable Calibration Standard"
self.prepopulatedFields[
KitsFieldNames.BASE_RECALIBRATION_MODES]["blind_recal"] = {
"text":
_("workflow.step.kits.base_recalibration_modes.blind_recal"),
"title":
_("workflow.step.kits.base_recalibration_modes.blind_recal.title"
),
} # "Blind Calibration"
self.prepopulatedFields[
KitsFieldNames.BASE_RECALIBRATION_MODES]["no_recal"] = {
"text":
_("workflow.step.kits.base_recalibration_modes.no_recal"),
"title":
_("workflow.step.kits.base_recalibration_modes.no_recal.title"
),
} # "No Calibration"
self.savedFields[KitsFieldNames.REALIGN] = gc.realign
self.savedFields[KitsFieldNames.FLOWS] = 0
self.savedFields[KitsFieldNames.LIBRARY_READ_LENGTH] = 0
self.savedFields[KitsFieldNames.READ_LENGTH] = 0
self.prepopulatedFields[
KitsFieldNames.IS_BARCODE_KIT_SELECTION_REQUIRED] = False
self.prepopulatedFields[KitsFieldNames.TEMPLATING_SIZE_CHOICES] = [
"200", "400"
]
self.savedFields[KitsFieldNames.TEMPLATING_SIZE] = ""
# For raptor templating kits, templating size cannot be used to drive UI behavior or db persistence. Use read length instead.
self.prepopulatedFields[KitsFieldNames.READ_LENGTH_CHOICES] = [
"200", "400"
]
self.prepopulatedFields[
KitsFieldNames.FLOWS_FROM_CATEGORY_RULES] = json.dumps(
KitInfo._category_flowCount_rules)
self.savedFields[KitsFieldNames.SAMPLE_PREP_PROTOCOL] = None
self.prepopulatedFields[
KitsFieldNames.SAMPLE_PREP_PROTOCOLS] = common_CV.objects.filter(
isActive=True, cv_type="samplePrepProtocol").order_by("uid")
self.prepopulatedFields[KitsFieldNames.PLAN_CATEGORIES] = ""
self.savedFields[KitsFieldNames.ADVANCED_SETTINGS_CHOICE] = "default"
self.prepopulatedFields[KitsFieldNames.ADVANCED_SETTINGS] = "{}"
self.sh_type = sh_type
def fileserver_space_check():
'''For each file server, compare disk usage to backup threshold value.
If disk usage exceeds threshold, launch celery task to delete/archive
raw data directories.
'''
# Get GlobalConfig object in order to access auto-acknowledge bit
gc = GlobalConfig.get()
auto_acknowledge = gc.auto_archive_ack
auto_action_enabled = gc.auto_archive_enable
# Get list of File Server objects
file_servers = FileServer.objects.all().order_by('pk').values()
# Get list of Report Storage objects
report_storages = ReportStorage.objects.all().order_by('pk').values()
# dict of fileset cateogries each with list of partition ids that can be acted upon.
category_list = {}
#-------------------------------------------------
# DELETE action only happens if threshold reached
#-------------------------------------------------
for dmfileset in DMFileSet.objects.filter(
version=settings.RELVERSION).filter(auto_action='DEL').values():
cat_name = slugify(dmfileset['type'])
category_list[cat_name] = {
'dmfileset': dmfileset,
'devlist': [],
'partitions': [],
}
for partition in _partitions(file_servers, report_storages):
if partition['diskusage'] >= dmfileset['auto_trigger_usage']:
logger.info("%s %s %.2f%% exceeds %s %.0f%%" %
(hex(partition['devid']), partition['path'],
partition['diskusage'], dmfileset['type'],
dmfileset['auto_trigger_usage']))
category_list[cat_name]['devlist'].append(partition['devid'])
category_list[cat_name]['partitions'].append(partition)
else:
logger.info("%s %s %.2f%% below %s %.0f%%" %
(hex(partition['devid']), partition['path'],
partition['diskusage'], dmfileset['type'],
dmfileset['auto_trigger_usage']))
# uniquify the deviceid list
category_list[cat_name]['devlist'] = list(
set(category_list[cat_name]['devlist']))
#-------------------------------------------------------------------------------
#ARCHIVE action happens as soon as grace period has expired (no threshold check)
#-------------------------------------------------------------------------------
for dmfileset in DMFileSet.objects.filter(
version=settings.RELVERSION).filter(auto_action='ARC').values():
cat_name = slugify(dmfileset['type'])
category_list[cat_name] = {
'dmfileset': dmfileset,
'devlist': [],
'partitions': [],
}
for partition in _partitions(file_servers, report_storages):
logger.debug("%s %s" %
(partition['path'], hex(partition['devid'])))
category_list[cat_name]['devlist'].append(partition['devid'])
category_list[cat_name]['partitions'].append(partition)
# uniquify the deviceid list
category_list[cat_name]['devlist'] = list(
set(category_list[cat_name]['devlist']))
#-------------------------------------------------------------
# Action loop - for each category, launch action per deviceid
#-------------------------------------------------------------
if not auto_action_enabled:
logger.info("Data management auto-action is disabled.")
# update any DMFileStats that are in use by active analysis jobs
try:
update_files_in_use()
except:
logger.error('Unable to update active DMFileStats')
logger.error(traceback.format_exc())
# Checks for manual export and archive requests.
manage_manual_action.delay()
for category, dict in category_list.iteritems():
for deviceid in dict['devlist']:
pathlist = [
item['path'] for item in dict['partitions']
if item['devid'] == deviceid
]
async_task_result = manage_data.delay(deviceid, dict['dmfileset'],
pathlist, auto_acknowledge,
auto_action_enabled)
return
def dashboard_fragments(request):
""" Returns the dashboard sections as html in a json object"""
time_span = request.GET.get("time_span", "24hours")
now = datetime.datetime.now(pytz.UTC)
DASHBOARD_TIME_SPANS = {
"hour": now - datetime.timedelta(hours=1),
"today": now.replace(hour=0, minute=0, second=0, microsecond=0),
"24hours": now - datetime.timedelta(hours=24),
"7days": now - datetime.timedelta(days=7),
# Used for testing only. Do not expose to the UI.
"__all__": datetime.datetime(year=1971, month=1, day=1),
}
if time_span not in DASHBOARD_TIME_SPANS:
raise Http404("Time span %s not available!" % time_span)
# runs section
runs = get_runs_list(DASHBOARD_TIME_SPANS[time_span])
runs_context = {
# Runs Section
"runs": {
"time_span": time_span,
"stages": DASHBOARD_STAGES,
"runs": runs,
},
}
# software update
update_status = GlobalConfig.get().ts_update_status
# services
services_down = []
for process, state in process_set():
if not state:
services_down.append(process)
show_cluster = False
nodes_down = []
if Cruncher.objects.count() > 0:
show_cluster = True
nodes_down = Cruncher.objects.exclude(state='G').values_list('name',
flat=True)
# data management
disk_usage = {}
for fs in FileServer.objects.all().order_by('pk'):
if os.path.exists(fs.filesPrefix):
disk_usage[fs.filesPrefix] = fs.percentfull
dm_active_jobs = DMFileStat.objects.filter(
action_state__in=['AG', 'DG', 'EG', 'SA', 'SE', 'SD', 'IG'
]).values_list('action_state', flat=True)
# instruments
rigs = Rig.objects.exclude(host_address='')
num_rigs = len(rigs)
if num_rigs > 1:
pool = Pool(processes=min(num_rigs, 20))
instruments = pool.map(get_instrument_info, rigs)
else:
instruments = [get_instrument_info(rig) for rig in rigs]
instr_connected = sum(
[instr['status'] == CONNECTED for instr in instruments])
instr_offline = sum([instr['status'] == OFFLINE for instr in instruments])
instr_alarm = sum([instr['status'] == ALARM for instr in instruments])
summary_context = {
# Summary Section
"summary": {
"ts_version": TS_version,
"update_status": update_status,
"instruments": {
"connected": instr_connected,
"offline": instr_offline,
"alerts": instr_alarm,
},
"services": {
"url": reverse("configure_services"),
"number_services_down": len(services_down),
"services_down": services_down,
"show_cluster": True if show_cluster else False,
"number_nodes_down": len(nodes_down) if show_cluster else "",
},
"data_management": {
"url":
reverse("datamanagement"),
"disk_usage":
disk_usage,
"show_path":
len(disk_usage) > 1,
"dm_jobs": [
("archive in progress",
sum([s == 'AG' for s in dm_active_jobs])),
("export in progress",
sum([s == 'EG' for s in dm_active_jobs])),
("delete in progress",
sum([s == 'DG' for s in dm_active_jobs])),
("import in progress",
sum([s == 'IG' for s in dm_active_jobs])),
("archive pending",
sum([s == 'SA' for s in dm_active_jobs])),
("export pending", sum([s == 'SE'
for s in dm_active_jobs])),
("delete pending", sum([s == 'SD'
for s in dm_active_jobs])),
]
}
},
}
instruments_context = {
"instruments":
sorted(instruments, key=lambda x: (x['status'], x['name'].lower()))
}
return HttpResponse(json.dumps({
"summary":
render_to_string("rundb/dashboard/fragments/summary.html",
summary_context),
"runs":
render_to_string("rundb/dashboard/fragments/runs.html", runs_context),
"instruments":
render_to_string("rundb/dashboard/fragments/instruments.html",
instruments_context)
}),
content_type="application/json")
def isOCP_enabled():
return GlobalConfig.get().enable_compendia_OCP
def __init__(self, sh_type):
super(KitsStepData, self).__init__(sh_type)
self.resourcePath = 'rundb/plan/page_plan/page_plan_kits.html'
self.prev_step_url = reverse("page_plan_application")
self.next_step_url = reverse("page_plan_plugins")
# 20130827-test
# self._dependsOn.append(StepNames.IONREPORTER)
self._dependsOn.append(StepNames.APPLICATION)
self._dependsOn.append(StepNames.BARCODE_BY_SAMPLE)
self.savedFields[KitsFieldNames.SAMPLE_PREPARATION_KIT] = None
self.prepopulatedFields[KitsFieldNames.SAMPLE_PREP_KITS] = KitInfo.objects.filter(kitType='SamplePrepKit', isActive=True).order_by('description')
self.savedFields[KitsFieldNames.LIBRARY_KIT_NAME] = None
self.prepopulatedFields[KitsFieldNames.LIB_KITS] = KitInfo.objects.filter(kitType__in=["LibraryKit", "LibraryPrepKit"], isActive=True).order_by("description")
self.savedFields[KitsFieldNames.LIBRARY_KEY] = None
self.prepopulatedFields[KitsFieldNames.FORWARD_LIB_KEYS] = LibraryKey.objects.filter(direction='Forward', runMode='single').order_by('-isDefault', 'name')
self.savedFields[KitsFieldNames.LIBRARY_KEY] = self.prepopulatedFields[KitsFieldNames.FORWARD_LIB_KEYS][0].sequence
self.savedFields[KitsFieldNames.TF_KEY] = GlobalConfig.get().default_test_fragment_key
self.savedFields[KitsFieldNames.FORWARD_3_PRIME_ADAPTER] = None
self.prepopulatedFields[KitsFieldNames.FORWARD_3_ADAPTERS] = ThreePrimeadapter.objects.filter(direction='Forward', runMode='single').order_by('-isDefault', 'chemistryType', 'name')
self.savedFields[KitsFieldNames.FORWARD_3_PRIME_ADAPTER] = self.prepopulatedFields[KitsFieldNames.FORWARD_3_ADAPTERS][0].sequence
self.savedFields[KitsFieldNames.FLOW_ORDER] = None
self.prepopulatedFields[KitsFieldNames.FLOW_ORDERS] = FlowOrder.objects.filter(isActive=True).order_by('-isDefault', 'description')
self.savedFields[KitsFieldNames.FLOW_ORDER] = None
self.savedFields[KitsFieldNames.TEMPLATE_KIT_NAME] = None
# no longer default to OneTouch
# self.savedFields[KitsFieldNames.TEMPLATE_KIT_TYPE] = KitsFieldNames.ONE_TOUCH
self.savedFields[KitsFieldNames.TEMPLATE_KIT_TYPE] = None
oneTouchDict = {
KitsFieldNames.KIT_VALUES: KitInfo.objects.filter(kitType__in=['TemplatingKit', 'AvalancheTemplateKit'], isActive=True).exclude(samplePrep_instrumentType="IA").order_by("description"),
KitsFieldNames.APPLICATION_DEFAULT: None
}
isoAmpDict = {
KitsFieldNames.KIT_VALUES: KitInfo.objects.filter(kitType__in=['TemplatingKit'], isActive=True, samplePrep_instrumentType="IA").order_by("description"),
KitsFieldNames.APPLICATION_DEFAULT: None
}
ionChefDict = {
KitsFieldNames.KIT_VALUES: KitInfo.objects.filter(kitType='IonChefPrepKit', isActive=True).order_by("description"),
KitsFieldNames.APPLICATION_DEFAULT: None
}
self.prepopulatedFields[KitsFieldNames.TEMPLATE_KIT_TYPES] = OrderedDict([
(KitsFieldNames.ONE_TOUCH, oneTouchDict),
(KitsFieldNames.ION_CHEF, ionChefDict),
(KitsFieldNames.ISO_AMP, isoAmpDict)
])
self.savedFields[KitsFieldNames.SEQUENCE_KIT_NAME] = None
self.prepopulatedFields[KitsFieldNames.SEQ_KITS] = KitInfo.objects.filter(kitType='SequencingKit', isActive=True).order_by("description")
self.savedFields[KitsFieldNames.TEMPLATE_KIT_NAME] = None
self.prepopulatedFields[KitsFieldNames.TEMPLATE_KITS] = KitInfo.objects.filter(kitType__in=['TemplatingKit', 'AvalancheTemplateKit', 'IonChefPrepKit'], isActive=True).order_by("description")
self.savedFields[KitsFieldNames.CONTROL_SEQUENCE] = None
self.prepopulatedFields[KitsFieldNames.CONTROL_SEQ_KITS] = KitInfo.objects.filter(kitType='ControlSequenceKit', isActive=True).order_by("description")
self.savedFields[KitsFieldNames.CHIP_TYPE] = None
self.prepopulatedFields[KitsFieldNames.INSTRUMENT_TYPES] = [(k, v) for k, v in Chip.ALLOWED_INSTRUMENT_TYPES if k]
self.prepopulatedFields[KitsFieldNames.CHIP_TYPES] = list(Chip.objects.filter(isActive=True).order_by('description', 'name').distinct('description'))
self.savedFields[KitsFieldNames.BARCODE_ID] = None
self.prepopulatedFields[KitsFieldNames.BARCODES] = list(dnaBarcode.objects.filter(active=True).values('name').distinct().order_by('name'))
gc = GlobalConfig.get()
self.savedFields[KitsFieldNames.IS_DUPLICATED_READS] = gc.mark_duplicates
self.savedFields[KitsFieldNames.BASE_RECALIBRATE] = gc.base_recalibration_mode
self.prepopulatedFields[KitsFieldNames.BASE_RECALIBRATION_MODES] = OrderedDict()
self.prepopulatedFields[KitsFieldNames.BASE_RECALIBRATION_MODES]["standard_recal"] = "Default Calibration"
self.prepopulatedFields[KitsFieldNames.BASE_RECALIBRATION_MODES]["panel_recal"] = "Enable Calibration Standard"
self.prepopulatedFields[KitsFieldNames.BASE_RECALIBRATION_MODES]["blind_recal"] = "Blind Calibration"
self.prepopulatedFields[KitsFieldNames.BASE_RECALIBRATION_MODES]["no_recal"] = "No Calibration"
self.savedFields[KitsFieldNames.REALIGN] = gc.realign
self.savedFields[KitsFieldNames.FLOWS] = 0
self.savedFields[KitsFieldNames.LIBRARY_READ_LENGTH] = 0
self.savedFields[KitsFieldNames.READ_LENGTH] = 0
self.prepopulatedFields[KitsFieldNames.IS_BARCODE_KIT_SELECTION_REQUIRED] = False
self.prepopulatedFields[KitsFieldNames.TEMPLATING_SIZE_CHOICES] = ["200", "400"]
self.savedFields[KitsFieldNames.TEMPLATING_SIZE] = ""
# For raptor templating kits, templating size cannot be used to drive UI behavior or db persistence. Use read length instead.
self.prepopulatedFields[KitsFieldNames.READ_LENGTH_CHOICES] = ["200", "400"]
self.prepopulatedFields[KitsFieldNames.FLOWS_FROM_CATEGORY_RULES] = json.dumps(KitInfo._category_flowCount_rules)
self.savedFields[KitsFieldNames.SAMPLE_PREP_PROTOCOL] = None
self.prepopulatedFields[KitsFieldNames.SAMPLE_PREP_PROTOCOLS] = common_CV.objects.filter(isActive=True, cv_type = "samplePrepProtocol").order_by('uid')
self.prepopulatedFields[KitsFieldNames.PLAN_CATEGORIES] = ""
self.savedFields[KitsFieldNames.ADVANCED_SETTINGS_CHOICE] = "default"
self.prepopulatedFields[KitsFieldNames.ADVANCED_SETTINGS] = "{}"
self.sh_type = sh_type
def dashboard_fragments(request, skip_runs=False):
""" Returns the dashboard sections as html in a json object"""
time_span = request.GET.get("time_span", "24hours")
now = datetime.datetime.now(pytz.UTC)
DASHBOARD_TIME_SPANS = {
"hour": now - datetime.timedelta(hours=1),
"today": now.replace(hour=0, minute=0, second=0, microsecond=0),
"24hours": now - datetime.timedelta(hours=24),
"7days": now - datetime.timedelta(days=7),
# Used for testing only. Do not expose to the UI.
"__all__": datetime.datetime(year=1971, month=1, day=1),
}
if time_span not in DASHBOARD_TIME_SPANS:
raise Http404("Time span %s not available!" % time_span)
# runs section
if skip_runs:
runs_context = {
# Runs Section
"runs": {
"time_span": time_span,
"stages": DASHBOARD_STAGES,
"runs": [],
"error": ""
},
}
else:
try:
runs = get_runs_list(DASHBOARD_TIME_SPANS[time_span])
runs_error = None
except Exception as err:
runs = []
runs_error = str(err)
runs_context = {
# Runs Section
"runs": {
"time_span": time_span,
"stages": DASHBOARD_STAGES,
"runs": runs,
"error": runs_error
},
}
# software update
update_status = GlobalConfig.get().ts_update_status
# services
services_down = []
for process, state in process_set():
if not state:
services_down.append(process)
show_cluster = False
nodes_down = []
if Cruncher.objects.count() > 0:
show_cluster = True
nodes_down = Cruncher.objects.exclude(state='G').values_list('name', flat=True)
# storage status
storage = get_storage_status()
# data management
disk_usage = get_disk_usage()
dm_active_jobs = DMFileStat.objects.filter(action_state__in=['AG', 'DG', 'EG', 'SA', 'SE', 'SD', 'IG']).values_list(
'action_state', flat=True)
# instruments
rigs = Rig.objects.exclude(host_address='')
num_rigs = len(rigs)
if num_rigs > 1:
pool = Pool(processes=min(num_rigs, 50))
instruments = pool.map(get_instrument_info, rigs)
else:
instruments = [get_instrument_info(rig) for rig in rigs]
instr_connected = sum([instr['status'] == CONNECTED for instr in instruments])
instr_offline = sum([instr['status'] == OFFLINE for instr in instruments])
instr_alarm = sum([instr['status'] == ALARM for instr in instruments])
summary_context = {
# Summary Section
"summary": {
"ts_version": TS_version,
"update_status": update_status,
"instruments": {
"connected": instr_connected,
"offline": instr_offline,
"alerts": instr_alarm,
},
"services": {
"url": reverse("configure_services"),
"number_services_down": len(services_down),
"services_down": services_down,
"show_cluster": True if show_cluster else False,
"number_nodes_down": len(nodes_down) if show_cluster else "",
"show_nas": storage['show_nas'],
"nas_status": storage.get('nas_status', ''),
"show_raid": storage['show_raid'],
"raid_status": storage.get('raid_status', ''),
},
"data_management": {
"url": reverse("datamanagement"),
"disk_usage": disk_usage,
"show_path": len(disk_usage) > 1,
"dm_jobs": [
("archive in progress", sum([s == 'AG' for s in dm_active_jobs])),
("export in progress", sum([s == 'EG' for s in dm_active_jobs])),
("delete in progress", sum([s == 'DG' for s in dm_active_jobs])),
("import in progress", sum([s == 'IG' for s in dm_active_jobs])),
("archive pending", sum([s == 'SA' for s in dm_active_jobs])),
("export pending", sum([s == 'SE' for s in dm_active_jobs])),
("delete pending", sum([s == 'SD' for s in dm_active_jobs])),
]
}
},
}
instruments_context = {
"instruments": sorted(instruments, key=lambda x: (x['status'], x['name'].lower()))
}
return HttpResponse(json.dumps({
"summary": render_to_string("rundb/home/fragments/summary.html", summary_context),
"runs": render_to_string("rundb/home/fragments/runs.html", runs_context),
"instruments": render_to_string("rundb/home/fragments/instruments.html", instruments_context)
}), content_type="application/json")
def __init__(self, sh_type):
super(KitsStepData, self).__init__(sh_type)
self.resourcePath = 'rundb/plan/page_plan/page_plan_kits.html'
self.prev_step_url = reverse("page_plan_application")
self.next_step_url = reverse("page_plan_plugins")
# 20130827-test
# self._dependsOn.append(StepNames.IONREPORTER)
self._dependsOn.append(StepNames.APPLICATION)
self._dependsOn.append(StepNames.BARCODE_BY_SAMPLE)
self.savedFields[KitsFieldNames.SAMPLE_PREPARATION_KIT] = None
self.prepopulatedFields[
KitsFieldNames.SAMPLE_PREP_KITS] = KitInfo.objects.filter(
kitType='SamplePrepKit', isActive=True).order_by('description')
self.savedFields[KitsFieldNames.LIBRARY_KIT_NAME] = None
self.prepopulatedFields[
KitsFieldNames.LIB_KITS] = KitInfo.objects.filter(
kitType__in=["LibraryKit", "LibraryPrepKit"],
isActive=True).order_by("description")
self.savedFields[KitsFieldNames.LIBRARY_KEY] = None
self.prepopulatedFields[
KitsFieldNames.FORWARD_LIB_KEYS] = LibraryKey.objects.filter(
direction='Forward',
runMode='single').order_by('-isDefault', 'name')
self.savedFields[KitsFieldNames.LIBRARY_KEY] = self.prepopulatedFields[
KitsFieldNames.FORWARD_LIB_KEYS][0].sequence
self.savedFields[KitsFieldNames.TF_KEY] = GlobalConfig.get(
).default_test_fragment_key
self.savedFields[KitsFieldNames.FORWARD_3_PRIME_ADAPTER] = None
self.prepopulatedFields[
KitsFieldNames.
FORWARD_3_ADAPTERS] = ThreePrimeadapter.objects.filter(
direction='Forward',
runMode='single').order_by('-isDefault', 'chemistryType',
'name')
self.savedFields[
KitsFieldNames.FORWARD_3_PRIME_ADAPTER] = self.prepopulatedFields[
KitsFieldNames.FORWARD_3_ADAPTERS][0].sequence
self.savedFields[KitsFieldNames.FLOW_ORDER] = None
self.prepopulatedFields[
KitsFieldNames.FLOW_ORDERS] = FlowOrder.objects.filter(
isActive=True).order_by('-isDefault', 'description')
self.savedFields[KitsFieldNames.FLOW_ORDER] = None
self.savedFields[
KitsFieldNames.AVALANCHE_FORWARD_3_PRIME_ADAPTER] = None
self.prepopulatedFields[
KitsFieldNames.
AVALANCHE_FORWARD_3_PRIME_ADAPTERS] = ThreePrimeadapter.objects.filter(
direction='Forward',
runMode='single',
chemistryType='avalanche').order_by('-isDefault', 'name')
self.savedFields[
KitsFieldNames.
AVALANCHE_FORWARD_3_PRIME_ADAPTER] = self.prepopulatedFields[
KitsFieldNames.AVALANCHE_FORWARD_3_PRIME_ADAPTERS][0].sequence
self.savedFields[KitsFieldNames.TEMPLATE_KIT_NAME] = None
# no longer default to OneTouch
# self.savedFields[KitsFieldNames.TEMPLATE_KIT_TYPE] = KitsFieldNames.ONE_TOUCH
self.savedFields[KitsFieldNames.TEMPLATE_KIT_TYPE] = None
oneTouchDict = {
KitsFieldNames.KIT_VALUES:
KitInfo.objects.filter(
kitType__in=['TemplatingKit', 'AvalancheTemplateKit'],
isActive=True).order_by("description"),
KitsFieldNames.APPLICATION_DEFAULT:
None
}
ionChefDict = {
KitsFieldNames.KIT_VALUES:
KitInfo.objects.filter(kitType='IonChefPrepKit',
isActive=True).order_by("description"),
KitsFieldNames.APPLICATION_DEFAULT:
None
}
oneTouchAvalancheDict = {
KitsFieldNames.KIT_VALUES:
KitInfo.objects.filter(kitType__in=['AvalancheTemplateKit'],
isActive=True).order_by("description"),
KitsFieldNames.APPLICATION_DEFAULT:
None
}
self.prepopulatedFields[KitsFieldNames.TEMPLATE_KIT_TYPES] = {
KitsFieldNames.ONE_TOUCH: oneTouchDict,
KitsFieldNames.ION_CHEF: ionChefDict,
KitsFieldNames.ONE_TOUCH_AVALANCHE: oneTouchAvalancheDict
}
self.savedFields[KitsFieldNames.SEQUENCE_KIT_NAME] = None
self.prepopulatedFields[
KitsFieldNames.SEQ_KITS] = KitInfo.objects.filter(
kitType='SequencingKit', isActive=True).order_by("description")
self.savedFields[KitsFieldNames.TEMPLATE_KIT_NAME] = None
self.prepopulatedFields[
KitsFieldNames.TEMPLATE_KITS] = KitInfo.objects.filter(
kitType__in=[
'TemplatingKit', 'AvalancheTemplateKit', 'IonChefPrepKit'
],
isActive=True).order_by("description")
self.savedFields[KitsFieldNames.CONTROL_SEQUENCE] = None
self.prepopulatedFields[
KitsFieldNames.CONTROL_SEQ_KITS] = KitInfo.objects.filter(
kitType='ControlSequenceKit',
isActive=True).order_by("description")
self.savedFields[KitsFieldNames.CHIP_TYPE] = None
self.prepopulatedFields[KitsFieldNames.INSTRUMENT_TYPES] = [
(k, v) for k, v in Chip.ALLOWED_INSTRUMENT_TYPES if k
]
self.prepopulatedFields[KitsFieldNames.CHIP_TYPES] = list(
Chip.objects.filter(isActive=True).order_by(
'description', 'name').distinct('description'))
self.savedFields[KitsFieldNames.BARCODE_ID] = None
self.prepopulatedFields[KitsFieldNames.BARCODES] = list(
dnaBarcode.objects.filter(
active=True).values('name').distinct().order_by('name'))
gc = GlobalConfig.get()
self.savedFields[
KitsFieldNames.IS_DUPLICATED_READS] = gc.mark_duplicates
self.savedFields[
KitsFieldNames.BASE_RECALIBRATE] = gc.base_recalibration_mode
self.prepopulatedFields[
KitsFieldNames.BASE_RECALIBRATION_MODES] = OrderedDict()
self.prepopulatedFields[KitsFieldNames.BASE_RECALIBRATION_MODES][
"standard_recal"] = "Default Calibration"
self.prepopulatedFields[KitsFieldNames.BASE_RECALIBRATION_MODES][
"panel_recal"] = "Enable Calibration Standard"
self.prepopulatedFields[KitsFieldNames.BASE_RECALIBRATION_MODES][
"blind_recal"] = "Blind Calibration"
self.prepopulatedFields[KitsFieldNames.BASE_RECALIBRATION_MODES][
"no_recal"] = "No Calibration"
self.savedFields[KitsFieldNames.REALIGN] = gc.realign
self.savedFields[KitsFieldNames.FLOWS] = 0
self.savedFields[KitsFieldNames.LIBRARY_READ_LENGTH] = 0
self.savedFields[KitsFieldNames.READ_LENGTH] = 0
self.prepopulatedFields[
KitsFieldNames.IS_BARCODE_KIT_SELECTION_REQUIRED] = False
self.prepopulatedFields[KitsFieldNames.TEMPLATING_SIZE_CHOICES] = [
"200", "400"
]
self.savedFields[KitsFieldNames.TEMPLATING_SIZE] = ""
# For raptor templating kits, templating size cannot be used to drive UI behavior or db persistence. Use read length instead.
self.prepopulatedFields[KitsFieldNames.READ_LENGTH_CHOICES] = [
"200", "400"
]
self.prepopulatedFields[
KitsFieldNames.FLOWS_FROM_CATEGORY_RULES] = json.dumps(
KitInfo._category_flowCount_rules)
self.savedFields[KitsFieldNames.SAMPLE_PREP_PROTOCOL] = None
self.prepopulatedFields[
KitsFieldNames.SAMPLE_PREP_PROTOCOLS] = common_CV.objects.filter(
isActive=True, cv_type="samplePrepProtocol").order_by('uid')
self.sh_type = sh_type
