def test_get_projects_new(self):
newProjects = ','.join([' a ', 'b', '', ' ', ' a b '])
data = {'projects': [], 'newProjects': newProjects}
result = get_projects('ionadmin', data)
self.assertEqual(len(result), 3,
'Incorrect number of projects created')
Project.objects.get(name='a')
Project.objects.get(name='b')
Project.objects.get(name='a_b')
def test_get_projects_new(self):
newProjects = ','.join([' a ', 'b', '', ' ', ' a b '])
data = {'projects':[],
'newProjects': newProjects}
result = get_projects('ionadmin', data)
self.assertEqual(len(result), 3, 'Incorrect number of projects created')
Project.objects.get(name='a')
Project.objects.get(name='b')
Project.objects.get(name='a_b')
def test_get_projects_existing(self):
projects = test_project.bulk_get_or_create_create_multi_projects(self)
self.assertEqual(len(projects), 3, 'Incorrect number of projects created')
Project.objects.get(name='a')
Project.objects.get(name='b')
Project.objects.get(name='a_b')
projectNameAndIds = [str(p.id) + '|' + p.name for p in projects]
data = {'projects':projectNameAndIds,
'newProjects': []}
result = get_projects('ionadmin', data)
self.assertEqual(len(result), len(projects), 'Incorrect number of projects retrieved')
Project.objects.get(name='a')
Project.objects.get(name='b')
Project.objects.get(name='a_b')
def test_get_projects_existing(self):
projects = test_project.bulk_get_or_create_create_multi_projects(self)
self.assertEqual(len(projects), 3,
'Incorrect number of projects created')
Project.objects.get(name='a')
Project.objects.get(name='b')
Project.objects.get(name='a_b')
projectNameAndIds = [str(p.id) + '|' + p.name for p in projects]
data = {'projects': projectNameAndIds, 'newProjects': []}
result = get_projects('ionadmin', data)
self.assertEqual(len(result), len(projects),
'Incorrect number of projects retrieved')
Project.objects.get(name='a')
Project.objects.get(name='b')
Project.objects.get(name='a_b')
def save_plan_or_template(request, planOid):
"""
Saving new or edited plan/template to db (source: plan template wizard)
Editing a planned run from having 1 sample to 2 samples will result in one edited planned run and one new planned run
"""
def isReusable(submitIntent):
return not (submitIntent == 'savePlan' or submitIntent == 'updatePlan')
if request.method != 'POST':
logger.exception(format_exc())
return HttpResponse(json.dumps({
"error":
"Error, unsupported HTTP Request method (%s) for plan update." %
request.method
}),
mimetype="application/json")
# Process Inputs
# pylint:disable=E1103
json_data = simplejson.loads(request.raw_post_data)
submitIntent = json_data.get('submitIntent', '')
logger.debug(
'views.save_plan_or_template POST.raw_post_data... simplejson Data: "%s"'
% json_data)
logger.debug("views.save_plan_or_template submitIntent=%s" % submitIntent)
# saving Template or Planned Run
isReusable = isReusable(submitIntent)
runModeValue = json_data.get('runMode', 'single')
isPlanGroupValue = runModeValue == 'pe' and not isReusable
libraryKeyValue = json_data.get('libraryKey', '')
forward3primeAdapterValue = json_data.get('forward3primeAdapter', '')
msgvalue = 'Run Plan' if not isReusable else 'Template'
if runModeValue == 'pe':
return HttpResponse(json.dumps({
"error":
"Error, paired-end plan is no longer supported. %s will not be saved."
% (msgvalue)
}),
mimetype="application/html")
planDisplayedNameValue = json_data.get('planDisplayedName', '').strip()
noteValue = json_data.get('notes_workaround', '')
# perform server-side validation to avoid things falling through the crack
if not planDisplayedNameValue:
return HttpResponse(json.dumps(
{"error": "Error, please enter a %s Name." % (msgvalue)}),
mimetype="application/html")
if not is_valid_chars(planDisplayedNameValue):
return HttpResponse(json.dumps(
{"error":
"Error, %s Name" % (msgvalue) + ERROR_MSG_INVALID_CHARS}),
mimetype="application/html")
if not is_valid_length(planDisplayedNameValue, MAX_LENGTH_PLAN_NAME):
return HttpResponse(json.dumps({
"error":
"Error, %s Name" % (msgvalue) + ERROR_MSG_INVALID_LENGTH %
(str(MAX_LENGTH_PLAN_NAME))
}),
mimetype="application/html")
if noteValue:
if not is_valid_chars(noteValue):
return HttpResponse(json.dumps({
"error":
"Error, %s note" % (msgvalue) + ERROR_MSG_INVALID_CHARS
}),
mimetype="application/html")
if not is_valid_length(noteValue, MAX_LENGTH_NOTES):
return HttpResponse(json.dumps({
"error":
"Error, Note" + ERROR_MSG_INVALID_LENGTH %
(str(MAX_LENGTH_NOTES))
}),
mimetype="application/html")
# Projects
projectObjList = get_projects(request.user, json_data)
# IonReporterUploader configuration and samples
selectedPlugins = json_data.get('selectedPlugins', {})
IRconfigList = json_data.get('irConfigList', [])
IRU_selected = False
for uploader in selectedPlugins.values():
if 'ionreporteruploader' in uploader['name'].lower(
) and uploader['name'] != 'IonReporterUploader_V1_0':
IRU_selected = True
#if IRU is set to autoRun, user does not need to select the plugin explicitly. user could have set all IRU versions to autorun
IRU_autorun_count = 0
if not IRU_selected:
IRU_autoruns = Plugin.objects.filter(
name__icontains="IonReporter",
selected=True,
active=True,
autorun=True).exclude(
name__icontains="IonReporterUploader_V1_0").order_by('-name')
IRU_autorun_count = IRU_autoruns.count()
if IRU_autorun_count > 0:
IRU_selected = True
if IRU_selected:
samples_IRconfig = json_data.get('sample_irConfig', '')
if samples_IRconfig:
samples_IRconfig = ','.join(samples_IRconfig)
#generate UUID for unique setIds
id_uuid = {}
setids = [ir.get('setid', "") for ir in IRconfigList]
if setids:
for setid in set(setids):
if setid:
id_uuid[setid] = str(uuid.uuid4())
for ir_config in IRconfigList:
setid = ir_config.get('setid', '')
if setid:
ir_config['setid'] += '__' + id_uuid[setid]
if IRU_autorun_count > 0 and not samples_IRconfig:
#if more than one IRU version is set to autorun and user does not explicitly select one,
#gui shows workflow config for IRU v1.0
samples_IRconfig = json_data.get('samples_workaround', '')
# Samples
barcodeIdValue = json_data.get('barcodeId', '')
barcodedSamples = ''
sampleValidationErrorMsg = ''
sampleValidationErrorMsg_leadingChars = ''
sampleValidationErrorMsg_length = ''
# one Plan will be created per entry in sampleList
# samples for barcoded Plan have a separate field (barcodedSamples)
if isReusable:
# samples entered only when saving planned run (not template)
sampleList = ['']
elif barcodeIdValue:
# a barcode Set is selected
sampleList = ['']
bcSamplesValues = json_data.get('bcSamples_workaround', '')
bcDictionary = {}
bcId = ""
for token in bcSamplesValues.split(","):
if ((token.find("bcKey|")) == 0):
bcId, bcId_str = token.split("|")[1:]
else:
sample = token.strip()
if bcId and sample:
if not is_valid_chars(sample):
sampleValidationErrorMsg += sample + ', '
elif is_invalid_leading_chars(sample):
sampleValidationErrorMsg_leadingChars += sample + ", "
elif not is_valid_length(sample, MAX_LENGTH_SAMPLE_NAME):
sampleValidationErrorMsg_length += sample + ", "
bcDictionary.setdefault(sample,
{}).setdefault('barcodes',
[]).append(bcId_str)
bcId = ""
barcodedSamples = simplejson.dumps(bcDictionary)
logger.debug(
"views.save_plan_or_template after simplejson.dumps... barcodedSamples=%s;"
% (barcodedSamples))
if not bcDictionary:
transaction.rollback()
return HttpResponse(json.dumps({
"error":
"Error, please enter at least one barcode sample name."
}),
mimetype="application/html")
else:
# Non-barcoded samples
sampleList = []
if IRU_selected:
samples = samples_IRconfig
else:
samples = json_data.get('samples_workaround', '')
for sample in samples.split(','):
if sample.strip():
if not is_valid_chars(sample):
sampleValidationErrorMsg += sample + ', '
elif is_invalid_leading_chars(sample):
sampleValidationErrorMsg_leadingChars += sample + ", "
elif not is_valid_length(sample, MAX_LENGTH_SAMPLE_NAME):
sampleValidationErrorMsg_length += sample + ", "
else:
sampleList.append(sample)
logger.debug("views.save_plan_or_template sampleList=%s " %
(sampleList))
if len(
sampleList
) == 0 and not sampleValidationErrorMsg and not sampleValidationErrorMsg_leadingChars and not sampleValidationErrorMsg_length:
transaction.rollback()
return HttpResponse(json.dumps({
"error":
"Error, please enter a sample name for the run plan."
}),
mimetype="application/html")
# Samples validation
if sampleValidationErrorMsg or sampleValidationErrorMsg_leadingChars or sampleValidationErrorMsg_length:
message = ""
if sampleValidationErrorMsg:
message = "Error, sample name" + ERROR_MSG_INVALID_CHARS
message = message + ' <br>Please fix: ' + sampleValidationErrorMsg + '<br>'
if sampleValidationErrorMsg_leadingChars:
message = message + "Error, sample name" + ERROR_MSG_INVALID_LEADING_CHARS
message = message + ' <br>Please fix: ' + sampleValidationErrorMsg_leadingChars + '<br>'
if sampleValidationErrorMsg_length:
message = message + "Error, sample name" + ERROR_MSG_INVALID_LENGTH % (
str(MAX_LENGTH_SAMPLE_NAME))
message = message + ' <br>Please fix: ' + sampleValidationErrorMsg_length
transaction.rollback()
return HttpResponse(json.dumps({"error": message}),
mimetype="application/html")
selectedPluginsValue = json_data.get('selectedPlugins', {})
# end processing input data
# Edit/Create Plan(s)
if int(planOid) == 0:
edit_existing_plan = False
else:
edit_existing_plan = True
for i, sample in enumerate(sampleList):
logger.debug(
"...LOOP... views.save_plan_or_template SAMPLE=%s; isSystem=%s; isReusable=%s; isPlanGroup=%s "
% (sample.strip(), json_data["isSystem"], isReusable,
isPlanGroupValue))
# add IonReporter config values for each sample
if len(IRconfigList) > 0:
for uploader in selectedPluginsValue.values():
if 'ionreporteruploader' in uploader['name'].lower():
if len(IRconfigList) > 1 and not barcodeIdValue:
uploader['userInput'] = [IRconfigList[i]]
else:
uploader['userInput'] = IRconfigList
if len(sampleList) > 1:
inputPlanDisplayedName = planDisplayedNameValue + '_' + sample.strip(
)
else:
inputPlanDisplayedName = planDisplayedNameValue
selectedTemplatingKit = json_data.get('templatekitname', '')
samplePrepInstrumentType = json_data.get('samplePrepInstrumentType',
'')
if samplePrepInstrumentType == 'ionChef':
selectedTemplatingKit = json_data.get('templatekitionchefname', '')
#PDD-TODO: remove the x_ prefix. the x_ prefix is just a reminder what the obsolete attributes to remove during the next phase
kwargs = {
'planDisplayedName':
inputPlanDisplayedName,
"planName":
inputPlanDisplayedName.replace(' ', '_'),
'usePreBeadfind':
toBoolean(json_data['usePreBeadfind'], False),
'usePostBeadfind':
toBoolean(json_data['usePostBeadfind'], False),
'preAnalysis':
True,
'runType':
json_data['runType'],
'templatingKitName':
selectedTemplatingKit,
'controlSequencekitname':
json_data.get('controlsequence', ''),
'runMode':
runModeValue,
'isSystem':
toBoolean(json_data['isSystem'], False),
'isReusable':
isReusable,
'isPlanGroup':
isPlanGroupValue,
'username':
request.user.username,
'isFavorite':
toBoolean(json_data.get('isFavorite', 'False'), False),
'pairedEndLibraryAdapterName':
json_data.get('pairedEndLibraryAdapterName', ''),
'samplePrepKitName':
json_data.get('samplePrepKitName', ''),
'planStatus':
"planned",
'x_autoAnalyze':
True,
'x_barcodedSamples':
barcodedSamples,
'x_barcodeId':
barcodeIdValue,
'x_bedfile':
json_data.get('bedfile', ''),
'x_chipType':
json_data.get('chipType', ''),
'x_flows':
json_data.get('flows', None),
'x_forward3primeadapter':
forward3primeAdapterValue,
###'_isReverseRun': = self.isReverseRun
'x_library':
json_data.get('library', ''),
'x_libraryKey':
libraryKeyValue,
'x_librarykitname':
json_data.get('librarykitname', ''),
'x_notes':
noteValue,
'x_regionfile':
json_data.get('regionfile', ''),
'x_sample':
sample.strip().replace(' ', '_'),
'x_sampleDisplayedName':
sample.strip(),
'x_selectedPlugins':
selectedPluginsValue,
'x_sequencekitname':
json_data.get('sequencekitname', ''),
'x_variantfrequency':
json_data.get('variantfrequency', ''),
}
planTemplate = None
#if we're changing a plan from having 1 sample to say 2 samples, we need to UPDATE 1 plan and CREATE 1 plan!!
try:
if not edit_existing_plan:
planTemplate, extra_kwargs = PlannedExperiment.objects.save_plan(
-1, **kwargs)
else:
planTemplate, extra_kwargs = PlannedExperiment.objects.save_plan(
planOid, **kwargs)
edit_existing_plan = False
# Update QCtype thresholds
qcTypes = QCType.objects.all()
for qcType in qcTypes:
qc_threshold = json_data.get(qcType.qcName, '')
if qc_threshold:
# get existing PlannedExperimentQC if any
plannedExpQcs = PlannedExperimentQC.objects.filter(
plannedExperiment=planTemplate.id, qcType=qcType.id)
if len(plannedExpQcs) > 0:
for plannedExpQc in plannedExpQcs:
plannedExpQc.threshold = qc_threshold
plannedExpQc.save()
else:
kwargs = {
'plannedExperiment': planTemplate,
'qcType': qcType,
'threshold': qc_threshold
}
plannedExpQc = PlannedExperimentQC(**kwargs)
plannedExpQc.save()
# add/remove projects
if projectObjList:
#TODO: refactor this logic to simplify using django orm
projectNameList = [project.name for project in projectObjList]
for currentProject in planTemplate.projects.all():
if currentProject.name not in projectNameList:
planTemplate.projects.remove(currentProject)
for projectObj in projectObjList:
planTemplate.projects.add(projectObj)
else:
planTemplate.projects.clear()
except ValidationError, err:
transaction.rollback()
logger.exception(format_exc())
message = "Internal error while trying to save the plan. "
for msg in err.messages:
message += str(msg)
message += " "
return HttpResponse(json.dumps({"error": message}),
mimetype="application/json")
except Exception as excp:
transaction.rollback()
logger.exception(format_exc())
message = "Internal error while trying to save the plan. %s" % (
excp.message)
return HttpResponse(json.dumps({"error": message}),
mimetype="application/json")
def save_uploaded_plans_for_template(request):
"""add plans, with CSV validation"""
logger.info(request)
if request.method != 'POST':
logger.exception(format_exc())
transaction.rollback()
return HttpResponse(json.dumps({
"error":
"Error, unsupported HTTP Request method (%s) for saving plan upload."
% request.method
}),
mimetype="application/json")
postedfile = request.FILES['postedfile']
destination = tempfile.NamedTemporaryFile(delete=False)
for chunk in postedfile.chunks():
destination.write(chunk)
postedfile.close()
destination.close()
#check to ensure it is not empty
headerCheck = open(destination.name, "rU")
firstCSV = []
for firstRow in csv.reader(headerCheck):
firstCSV.append(firstRow)
#logger.info("views.save_uploaded_plans_for_template() firstRow=%s;" %(firstRow))
headerCheck.close()
if not firstRow:
os.unlink(destination.name)
transaction.rollback()
return HttpResponse(json.dumps(
{"status": "Error: batch planning file is empty"}),
mimetype="text/html")
index = 0
plans = []
rawPlanDataList = []
failed = {}
file = open(destination.name, "rU")
reader = csv.DictReader(file)
for index, row in enumerate(reader, start=1):
errorMsg, aPlanDict, rawPlanDict, isToSkipRow = validate_csv_plan(row)
logger.info(
"views.save_uploaded_plans_for_template() index=%d; errorMsg=%s; planDict=%s"
% (index, errorMsg, rawPlanDict))
if errorMsg:
logger.info(
"views.save_uploaded_plans_for_template() ERROR MESSAGE index=%d; errorMsg=%s; planDict=%s"
% (index, errorMsg, rawPlanDict))
failed[index] = errorMsg
continue
elif isToSkipRow:
logger.info(
"views.save_uploaded_plans_for_template() SKIPPED ROW index=%d; row=%s"
% (index, row))
continue
else:
plans.append(aPlanDict)
rawPlanDataList.append(rawPlanDict)
destination.close() # now close and remove the temp file
os.unlink(destination.name)
if index == 0:
transaction.rollback()
return HttpResponse(json.dumps({
"status":
"Error: There must be at least one plan! Please reload the page and try again with more plans."
}),
mimetype="text/html")
if failed:
r = {
"status": "Plan validation failed. The plans have not been saved.",
"failed": failed
}
logger.info("views.save_uploaded_plans_for_template() failed=%s" % (r))
transaction.rollback()
return HttpResponse(json.dumps(r), mimetype="text/html")
#saving to db needs to be the last thing to happen
try:
index = 0
for planFamily in plans:
plan = planFamily['plan']
plan.save()
expObj = planFamily['exp']
expObj.plan = plan
expObj.expName = plan.planGUID
expObj.unique = plan.planGUID
expObj.displayname = plan.planGUID
expObj.save()
easObj = planFamily['eas']
easObj.experiment = expObj
easObj.isEditable = True
easObj.save()
#saving/associating samples
sampleDisplayedNames = planFamily['samples']
sampleNames = [
name.replace(' ', '_') for name in sampleDisplayedNames
]
externalId = None
for name, displayedName in zip(sampleNames, sampleDisplayedNames):
sample_kwargs = {
'name': name,
'displayedName': displayedName,
'date': plan.date,
'status': plan.planStatus,
'externalId': externalId
}
sample = Sample.objects.get_or_create(
name=name, externalId=externalId,
defaults=sample_kwargs)[0]
sample.experiments.add(expObj)
sample.save()
planDict = rawPlanDataList[index]
# add QCtype thresholds
qcTypes = QCType.objects.all()
for qcType in qcTypes:
qc_threshold = planDict.get(qcType.qcName, '')
if qc_threshold:
# get existing PlannedExperimentQC if any
plannedExpQcs = PlannedExperimentQC.objects.filter(
plannedExperiment=plan.id, qcType=qcType.id)
if len(plannedExpQcs) > 0:
for plannedExpQc in plannedExpQcs:
plannedExpQc.threshold = qc_threshold
plannedExpQc.save()
else:
kwargs = {
'plannedExperiment': plan,
'qcType': qcType,
'threshold': qc_threshold
}
plannedExpQc = PlannedExperimentQC(**kwargs)
plannedExpQc.save()
# add projects
projectObjList = get_projects(request.user, planDict)
for project in projectObjList:
if project:
plan.projects.add(project)
index += 1
except:
logger.exception(format_exc())
transaction.rollback()
return HttpResponse(json.dumps(
{"status": "Error saving plans to database!"}),
mimetype="text/html")
##return HttpResponse(json.dumps({"error": "Internal error while trying to save the plan."}), mimetype="application/json")
else:
transaction.commit()
r = {
"status":
"Plans Uploaded! The plans will be listed on the planned run page.",
"failed": failed
}
return HttpResponse(json.dumps(r), mimetype="text/html")
def test_get_projects(self):
data = {'projects': [], 'newProjects': []}
# data = simplejson.loads(data)
result = get_projects('ionadmin', data)
self.assertFalse(result, 'list should be empty')
def save_plan_or_template(request, planOid):
"""
Saving new or edited plan/template to db (source: plan template wizard)
Editing a planned run from having 1 sample to 2 samples will result in one edited planned run and one new planned run
"""
def isReusable(submitIntent):
return not (submitIntent == 'savePlan' or submitIntent == 'updatePlan')
if request.method != 'POST':
logger.exception(format_exc())
return HttpResponse(json.dumps({"error": "Error, unsupported HTTP Request method (%s) for plan update." % request.method}), mimetype="application/json")
# Process Inputs
# pylint:disable=E1103
json_data = simplejson.loads(request.raw_post_data)
submitIntent = json_data.get('submitIntent', '')
logger.debug('views.save_plan_or_template POST.raw_post_data... simplejson Data: "%s"' % json_data)
logger.debug("views.save_plan_or_template submitIntent=%s" % submitIntent)
# saving Template or Planned Run
isReusable = isReusable(submitIntent)
runModeValue = json_data.get('runMode', 'single')
isPlanGroupValue = runModeValue == 'pe' and not isReusable
libraryKeyValue = json_data.get('libraryKey', '')
forward3primeAdapterValue = json_data.get('forward3primeAdapter', '')
msgvalue = 'Run Plan' if not isReusable else 'Template'
if runModeValue == 'pe':
return HttpResponse(json.dumps({"error": "Error, paired-end plan is no longer supported. %s will not be saved." % (msgvalue)}), mimetype="application/html")
planDisplayedNameValue = json_data.get('planDisplayedName', '').strip()
noteValue = json_data.get('notes_workaround', '')
# perform server-side validation to avoid things falling through the crack
if not planDisplayedNameValue:
return HttpResponse(json.dumps({"error": "Error, please enter a %s Name." %(msgvalue)}), mimetype="application/html")
if not is_valid_chars(planDisplayedNameValue):
return HttpResponse(json.dumps({"error": "Error, %s Name" %(msgvalue) + ERROR_MSG_INVALID_CHARS}), mimetype="application/html")
if not is_valid_length(planDisplayedNameValue, MAX_LENGTH_PLAN_NAME):
return HttpResponse(json.dumps({"error": "Error, %s Name" %(msgvalue) + ERROR_MSG_INVALID_LENGTH %(str(MAX_LENGTH_PLAN_NAME))}), mimetype="application/html")
if noteValue:
if not is_valid_chars(noteValue):
return HttpResponse(json.dumps({"error": "Error, %s note" %(msgvalue) + ERROR_MSG_INVALID_CHARS}), mimetype="application/html")
if not is_valid_length(noteValue, MAX_LENGTH_NOTES):
return HttpResponse(json.dumps({"error": "Error, Note" + ERROR_MSG_INVALID_LENGTH %(str(MAX_LENGTH_NOTES))}), mimetype="application/html")
# Projects
projectObjList = get_projects(request.user, json_data)
# IonReporterUploader configuration and samples
selectedPlugins = json_data.get('selectedPlugins', {})
IRconfigList = json_data.get('irConfigList', [])
IRU_selected = False
for uploader in selectedPlugins.values():
if 'ionreporteruploader' in uploader['name'].lower() and uploader['name'] != 'IonReporterUploader_V1_0':
IRU_selected = True
#if IRU is set to autoRun, user does not need to select the plugin explicitly. user could have set all IRU versions to autorun
IRU_autorun_count = 0
if not IRU_selected:
IRU_autoruns = Plugin.objects.filter(name__icontains="IonReporter", selected=True, active=True, autorun=True).exclude(name__icontains="IonReporterUploader_V1_0").order_by('-name')
IRU_autorun_count = IRU_autoruns.count()
if IRU_autorun_count > 0:
IRU_selected = True
if IRU_selected:
samples_IRconfig = json_data.get('sample_irConfig', '')
if samples_IRconfig:
samples_IRconfig = ','.join(samples_IRconfig)
#generate UUID for unique setIds
id_uuid = {}
setids = [ir.get('setid', "") for ir in IRconfigList]
if setids:
for setid in set(setids):
if setid:
id_uuid[setid] = str(uuid.uuid4())
for ir_config in IRconfigList:
setid = ir_config.get('setid', '')
if setid:
ir_config['setid'] += '__' + id_uuid[setid]
if IRU_autorun_count > 0 and not samples_IRconfig:
#if more than one IRU version is set to autorun and user does not explicitly select one,
#gui shows workflow config for IRU v1.0
samples_IRconfig = json_data.get('samples_workaround', '')
# Samples
barcodeIdValue = json_data.get('barcodeId', '')
barcodedSamples = ''
sampleValidationErrorMsg = ''
sampleValidationErrorMsg_leadingChars = ''
sampleValidationErrorMsg_length = ''
# one Plan will be created per entry in sampleList
# samples for barcoded Plan have a separate field (barcodedSamples)
if isReusable:
# samples entered only when saving planned run (not template)
sampleList = ['']
elif barcodeIdValue:
# a barcode Set is selected
sampleList = ['']
bcSamplesValues = json_data.get('bcSamples_workaround', '')
bcDictionary = {}
bcId = ""
for token in bcSamplesValues.split(","):
if ((token.find("bcKey|")) == 0):
bcId, bcId_str = token.split("|")[1:]
else:
sample = token.strip()
if bcId and sample:
if not is_valid_chars(sample):
sampleValidationErrorMsg += sample + ', '
elif is_invalid_leading_chars(sample):
sampleValidationErrorMsg_leadingChars += sample + ", "
elif not is_valid_length(sample, MAX_LENGTH_SAMPLE_NAME):
sampleValidationErrorMsg_length += sample + ", "
bcDictionary.setdefault(sample, {}).setdefault('barcodes',[]).append(bcId_str)
bcId = ""
barcodedSamples = simplejson.dumps(bcDictionary)
logger.debug("views.save_plan_or_template after simplejson.dumps... barcodedSamples=%s;" % (barcodedSamples))
if not bcDictionary:
transaction.rollback()
return HttpResponse(json.dumps({"error": "Error, please enter at least one barcode sample name."}), mimetype="application/html")
else:
# Non-barcoded samples
sampleList = []
if IRU_selected:
samples = samples_IRconfig
else:
samples = json_data.get('samples_workaround', '')
for sample in samples.split(','):
if sample.strip():
if not is_valid_chars(sample):
sampleValidationErrorMsg += sample + ', '
elif is_invalid_leading_chars(sample):
sampleValidationErrorMsg_leadingChars += sample + ", "
elif not is_valid_length(sample, MAX_LENGTH_SAMPLE_NAME):
sampleValidationErrorMsg_length += sample + ", "
else:
sampleList.append(sample)
logger.debug("views.save_plan_or_template sampleList=%s " % (sampleList))
if len(sampleList) == 0 and not sampleValidationErrorMsg and not sampleValidationErrorMsg_leadingChars and not sampleValidationErrorMsg_length:
transaction.rollback()
return HttpResponse(json.dumps({"error": "Error, please enter a sample name for the run plan."}), mimetype="application/html")
# Samples validation
if sampleValidationErrorMsg or sampleValidationErrorMsg_leadingChars or sampleValidationErrorMsg_length:
message = ""
if sampleValidationErrorMsg:
message = "Error, sample name" + ERROR_MSG_INVALID_CHARS
message = message + ' <br>Please fix: ' + sampleValidationErrorMsg + '<br>'
if sampleValidationErrorMsg_leadingChars:
message = message + "Error, sample name" + ERROR_MSG_INVALID_LEADING_CHARS
message = message + ' <br>Please fix: ' + sampleValidationErrorMsg_leadingChars + '<br>'
if sampleValidationErrorMsg_length:
message = message + "Error, sample name" + ERROR_MSG_INVALID_LENGTH %(str(MAX_LENGTH_SAMPLE_NAME))
message = message + ' <br>Please fix: ' + sampleValidationErrorMsg_length
transaction.rollback()
return HttpResponse(json.dumps({"error": message}), mimetype="application/html")
selectedPluginsValue = json_data.get('selectedPlugins', {})
# end processing input data
# Edit/Create Plan(s)
if int(planOid) == 0:
edit_existing_plan = False
else:
edit_existing_plan = True
for i, sample in enumerate(sampleList):
logger.debug("...LOOP... views.save_plan_or_template SAMPLE=%s; isSystem=%s; isReusable=%s; isPlanGroup=%s "
% (sample.strip(), json_data["isSystem"], isReusable, isPlanGroupValue))
# add IonReporter config values for each sample
if len(IRconfigList) > 0:
for uploader in selectedPluginsValue.values():
if 'ionreporteruploader' in uploader['name'].lower():
if len(IRconfigList) > 1 and not barcodeIdValue:
uploader['userInput'] = [IRconfigList[i]]
else:
uploader['userInput'] = IRconfigList
if len(sampleList) > 1:
inputPlanDisplayedName = planDisplayedNameValue + '_' + sample.strip()
else:
inputPlanDisplayedName = planDisplayedNameValue
selectedTemplatingKit = json_data.get('templatekitname', '')
samplePrepInstrumentType = json_data.get('samplePrepInstrumentType', '')
if samplePrepInstrumentType == 'ionChef':
selectedTemplatingKit = json_data.get('templatekitionchefname', '')
#PDD-TODO: remove the x_ prefix. the x_ prefix is just a reminder what the obsolete attributes to remove during the next phase
kwargs = {
'planDisplayedName': inputPlanDisplayedName,
"planName": inputPlanDisplayedName.replace(' ', '_'),
'usePreBeadfind': toBoolean(json_data['usePreBeadfind'], False),
'usePostBeadfind': toBoolean(json_data['usePostBeadfind'], False),
'preAnalysis': True,
'runType': json_data['runType'],
'templatingKitName': selectedTemplatingKit,
'controlSequencekitname': json_data.get('controlsequence', ''),
'runMode': runModeValue,
'isSystem': toBoolean(json_data['isSystem'], False),
'isReusable': isReusable,
'isPlanGroup': isPlanGroupValue,
'username': request.user.username,
'isFavorite': toBoolean(json_data.get('isFavorite', 'False'), False),
'pairedEndLibraryAdapterName': json_data.get('pairedEndLibraryAdapterName', ''),
'samplePrepKitName': json_data.get('samplePrepKitName', ''),
'planStatus' : "planned",
'x_autoAnalyze': True,
'x_barcodedSamples': barcodedSamples,
'x_barcodeId': barcodeIdValue,
'x_bedfile': json_data.get('bedfile', ''),
'x_chipType': json_data.get('chipType', ''),
'x_flows': json_data.get('flows', None),
'x_forward3primeadapter': forward3primeAdapterValue,
###'_isReverseRun': = self.isReverseRun
'x_library': json_data.get('library', ''),
'x_libraryKey': libraryKeyValue,
'x_librarykitname': json_data.get('librarykitname', ''),
'x_notes': noteValue,
'x_regionfile': json_data.get('regionfile', ''),
'x_sample': sample.strip().replace(' ', '_'),
'x_sampleDisplayedName': sample.strip(),
'x_selectedPlugins': selectedPluginsValue,
'x_sequencekitname': json_data.get('sequencekitname', ''),
'x_variantfrequency': json_data.get('variantfrequency', ''),
}
planTemplate = None
#if we're changing a plan from having 1 sample to say 2 samples, we need to UPDATE 1 plan and CREATE 1 plan!!
try:
if not edit_existing_plan:
planTemplate, extra_kwargs = PlannedExperiment.objects.save_plan(-1, **kwargs)
else:
planTemplate, extra_kwargs = PlannedExperiment.objects.save_plan(planOid, **kwargs)
edit_existing_plan = False
# Update QCtype thresholds
qcTypes = QCType.objects.all()
for qcType in qcTypes:
qc_threshold = json_data.get(qcType.qcName, '')
if qc_threshold:
# get existing PlannedExperimentQC if any
plannedExpQcs = PlannedExperimentQC.objects.filter(plannedExperiment=planTemplate.id, qcType=qcType.id)
if len(plannedExpQcs) > 0:
for plannedExpQc in plannedExpQcs:
plannedExpQc.threshold = qc_threshold
plannedExpQc.save()
else:
kwargs = {
'plannedExperiment': planTemplate,
'qcType': qcType,
'threshold': qc_threshold
}
plannedExpQc = PlannedExperimentQC(**kwargs)
plannedExpQc.save()
# add/remove projects
if projectObjList:
#TODO: refactor this logic to simplify using django orm
projectNameList = [project.name for project in projectObjList]
for currentProject in planTemplate.projects.all():
if currentProject.name not in projectNameList:
planTemplate.projects.remove(currentProject)
for projectObj in projectObjList:
planTemplate.projects.add(projectObj)
else:
planTemplate.projects.clear()
except ValidationError, err:
transaction.rollback()
logger.exception(format_exc())
message = "Internal error while trying to save the plan. "
for msg in err.messages:
message += str(msg)
message += " "
return HttpResponse(json.dumps({"error": message}), mimetype="application/json")
except Exception as excp:
transaction.rollback()
logger.exception(format_exc())
message = "Internal error while trying to save the plan. %s" %(excp.message)
return HttpResponse(json.dumps({"error": message}), mimetype="application/json")
def save_uploaded_plans_for_template(request):
"""add plans, with CSV validation"""
logger.info(request)
if request.method != 'POST':
logger.exception(format_exc())
transaction.rollback()
return HttpResponse(json.dumps({"error": "Error, unsupported HTTP Request method (%s) for saving plan upload." % request.method}), mimetype="application/json")
postedfile = request.FILES['postedfile']
destination = tempfile.NamedTemporaryFile(delete=False)
for chunk in postedfile.chunks():
destination.write(chunk)
postedfile.close()
destination.close()
#check to ensure it is not empty
headerCheck = open(destination.name, "rU")
firstCSV = []
for firstRow in csv.reader(headerCheck):
firstCSV.append(firstRow)
#logger.info("views.save_uploaded_plans_for_template() firstRow=%s;" %(firstRow))
headerCheck.close()
if not firstRow:
os.unlink(destination.name)
transaction.rollback()
return HttpResponse(json.dumps({"status": "Error: batch planning file is empty"}), mimetype="text/html")
index = 0
plans = []
rawPlanDataList = []
failed = {}
file = open(destination.name, "rU")
reader = csv.DictReader(file)
for index, row in enumerate(reader, start=1):
errorMsg, aPlanDict, rawPlanDict, isToSkipRow = validate_csv_plan(row)
logger.info("views.save_uploaded_plans_for_template() index=%d; errorMsg=%s; planDict=%s" %(index, errorMsg, rawPlanDict))
if errorMsg:
logger.info("views.save_uploaded_plans_for_template() ERROR MESSAGE index=%d; errorMsg=%s; planDict=%s" %(index, errorMsg, rawPlanDict))
failed[index] = errorMsg
continue
elif isToSkipRow:
logger.info("views.save_uploaded_plans_for_template() SKIPPED ROW index=%d; row=%s" %(index, row))
continue
else:
plans.append(aPlanDict)
rawPlanDataList.append(rawPlanDict)
destination.close() # now close and remove the temp file
os.unlink(destination.name)
if index == 0:
transaction.rollback()
return HttpResponse(json.dumps({"status": "Error: There must be at least one plan! Please reload the page and try again with more plans."}), mimetype="text/html")
if failed:
r = {"status": "Plan validation failed. The plans have not been saved.", "failed": failed}
logger.info("views.save_uploaded_plans_for_template() failed=%s" %(r))
transaction.rollback()
return HttpResponse(json.dumps(r), mimetype="text/html")
#saving to db needs to be the last thing to happen
try:
index = 0
for planFamily in plans:
plan = planFamily['plan']
plan.save()
expObj = planFamily['exp']
expObj.plan = plan
expObj.expName = plan.planGUID
expObj.unique = plan.planGUID
expObj.displayname = plan.planGUID
expObj.save()
easObj = planFamily['eas']
easObj.experiment = expObj
easObj.isEditable = True
easObj.save()
#saving/associating samples
sampleDisplayedNames = planFamily['samples']
sampleNames = [name.replace(' ', '_') for name in sampleDisplayedNames]
externalId = None
for name, displayedName in zip(sampleNames, sampleDisplayedNames):
sample_kwargs = {
'name' : name,
'displayedName' : displayedName,
'date' : plan.date,
'status' : plan.planStatus,
'externalId': externalId
}
sample = Sample.objects.get_or_create(name=name, externalId=externalId, defaults=sample_kwargs)[0]
sample.experiments.add(expObj)
sample.save()
planDict = rawPlanDataList[index]
# add QCtype thresholds
qcTypes = QCType.objects.all()
for qcType in qcTypes:
qc_threshold = planDict.get(qcType.qcName, '')
if qc_threshold:
# get existing PlannedExperimentQC if any
plannedExpQcs = PlannedExperimentQC.objects.filter(plannedExperiment=plan.id, qcType=qcType.id)
if len(plannedExpQcs) > 0:
for plannedExpQc in plannedExpQcs:
plannedExpQc.threshold = qc_threshold
plannedExpQc.save()
else:
kwargs = {
'plannedExperiment': plan,
'qcType': qcType,
'threshold': qc_threshold
}
plannedExpQc = PlannedExperimentQC(**kwargs)
plannedExpQc.save()
# add projects
projectObjList = get_projects(request.user, planDict)
for project in projectObjList:
if project:
plan.projects.add(project)
index += 1
except:
logger.exception(format_exc())
transaction.rollback()
return HttpResponse(json.dumps({"status": "Error saving plans to database!"}), mimetype="text/html")
##return HttpResponse(json.dumps({"error": "Internal error while trying to save the plan."}), mimetype="application/json")
else:
transaction.commit()
r = {"status": "Plans Uploaded! The plans will be listed on the planned run page.", "failed": failed}
return HttpResponse(json.dumps(r), mimetype="text/html")
def save_plan_or_template(request, planOid):
"""
Saving new or edited plan/template to db (source: plan template wizard)
Editing a planned run from having 1 sample to 2 samples will result in one edited planned run and one new planned run
"""
def isReusable(submitIntent):
return not (submitIntent == 'savePlan' or submitIntent == 'updatePlan')
def isValidChars(value, validChars=r'^[a-zA-Z0-9-_\.\s\,]+$'):
''' Determines if value is valid: letters, numbers, spaces, dashes, underscores only '''
return bool(re.compile(validChars).match(value))
if request.method != 'POST':
logger.exception(format_exc())
return HttpResponse(json.dumps({
"error":
"Error, unsupported HTTP Request method (%s) for plan update." %
request.method
}),
mimetype="application/json")
# Process Inputs
# pylint:disable=E1103
json_data = simplejson.loads(request.raw_post_data)
submitIntent = json_data.get('submitIntent', '')
logger.debug(
'views.editplannedexperiment POST.raw_post_data... simplejson Data: "%s"'
% json_data)
logger.debug("views.editplannedexperiment submitIntent=%s" % submitIntent)
# saving Template or Planned Run
isReusable = isReusable(submitIntent)
runModeValue = json_data.get('runMode', 'single')
isPlanGroupValue = runModeValue == 'pe' and not isReusable
libraryKeyValue = json_data.get('libraryKey', '')
forward3primeAdapterValue = json_data.get('forward3primeAdapter', '')
msgvalue = 'Run Plan' if not isReusable else 'Template'
if runModeValue == 'pe':
return HttpResponse(json.dumps({
"error":
"Error, paired-end plan is no longer supported. %s will not be saved."
% (msgvalue)
}),
mimetype="application/html")
planDisplayedNameValue = json_data.get('planDisplayedName', '').strip()
noteValue = json_data.get('notes_workaround', '')
# perform server-side validation to avoid things falling through the crack
if not planDisplayedNameValue:
return HttpResponse(json.dumps(
{"error": "Error, please enter a %s Name." % (msgvalue)}),
mimetype="application/html")
if not isValidChars(planDisplayedNameValue):
return HttpResponse(json.dumps({
"error":
"Error, %s Name should contain only numbers, letters, spaces, and the following: . - _"
% (msgvalue)
}),
mimetype="application/html")
if noteValue and not isValidChars(noteValue):
return HttpResponse(json.dumps({
"error":
"Error, %s note should contain only numbers, letters, spaces, and the following: . - _"
% (msgvalue)
}),
mimetype="application/html")
# Projects
projectObjList = get_projects(request.user, json_data)
# IonReporterUploader configuration and samples
selectedPlugins = json_data.get('selectedPlugins', {})
IRconfigList = json_data.get('irConfigList', [])
IRU_1_2_selected = False
for uploader in selectedPlugins.get('planuploaders', []):
if 'ionreporteruploader' in uploader['name'].lower(
) and uploader['name'] != 'IonReporterUploader_V1_0':
IRU_1_2_selected = True
samples_IRconfig = json_data.get('sample_irConfig', '')
samples_IRconfig = ','.join(samples_IRconfig)
#generate UUID for unique setIds
id_uuid = {}
setids = [ir['setid'] for ir in IRconfigList]
for setid in set(setids):
id_uuid[setid] = str(uuid.uuid4())
for ir_config in IRconfigList:
ir_config['setid'] += '__' + id_uuid[ir_config['setid']]
# Samples
barcodeIdValue = json_data.get('barcodeId', '')
barcodedSamples = ''
sampleValidationErrorMsg = ''
# one Plan will be created per entry in sampleList
# samples for barcoded Plan have a separate field (barcodedSamples)
if isReusable:
# samples entered only when saving planned run (not template)
sampleList = ['']
elif barcodeIdValue:
# a barcode Set is selected
sampleList = ['']
bcSamplesValues = json_data.get('bcSamples_workaround', '')
bcDictionary = {}
bcId = ""
for token in bcSamplesValues.split(","):
if ((token.find("bcKey|")) == 0):
bcId, bcId_str = token.split("|")[1:]
else:
sample = token.strip()
if bcId and sample:
if not isValidChars(sample):
sampleValidationErrorMsg += sample + ', '
bcDictionary.setdefault(sample,
{}).setdefault('barcodes',
[]).append(bcId_str)
bcId = ""
barcodedSamples = simplejson.dumps(bcDictionary)
logger.debug(
"views.editplannedexperiment after simplejson.dumps... barcodedSamples=%s;"
% (barcodedSamples))
if not bcDictionary:
transaction.rollback()
return HttpResponse(json.dumps({
"error":
"Error, please enter at least one barcode sample name."
}),
mimetype="application/html")
else:
# Non-barcoded samples
sampleList = []
if IRU_1_2_selected:
samples = samples_IRconfig
else:
samples = json_data.get('samples_workaround', '')
for sample in samples.split(','):
if sample.strip():
if not isValidChars(sample):
sampleValidationErrorMsg += sample + ', '
else:
sampleList.append(sample)
logger.debug("views.editplannedexperiment sampleList=%s " %
(sampleList))
if len(sampleList) == 0:
transaction.rollback()
return HttpResponse(json.dumps({
"error":
"Error, please enter a sample name for the run plan."
}),
mimetype="application/html")
# Samples validation
if sampleValidationErrorMsg:
message = "Error, sample name should contain only numbers, letters, spaces, and the following: . - _"
message = message + ' <br>Please fix: ' + sampleValidationErrorMsg
transaction.rollback()
return HttpResponse(json.dumps({"error": message}),
mimetype="application/html")
selectedPluginsValue = json_data.get('selectedPlugins', [])
# end processing input data
# Edit/Create Plan(s)
if int(planOid) == 0:
edit_existing_plan = False
else:
edit_existing_plan = True
for i, sample in enumerate(sampleList):
logger.debug(
"...LOOP... views.editplannedexperiment SAMPLE=%s; isSystem=%s; isReusable=%s; isPlanGroup=%s "
% (sample.strip(), json_data["isSystem"], isReusable,
isPlanGroupValue))
# add IonReporter config values for each sample
if len(IRconfigList) > 0:
for uploader in selectedPluginsValue['planuploaders']:
if 'ionreporter' in uploader['name'].lower():
if len(IRconfigList) > 1 and not barcodeIdValue:
uploader['userInput'] = [IRconfigList[i]]
else:
uploader['userInput'] = IRconfigList
if len(sampleList) > 1:
inputPlanDisplayedName = planDisplayedNameValue + '_' + sample.strip(
)
else:
inputPlanDisplayedName = planDisplayedNameValue
kwargs = {
'planDisplayedName':
inputPlanDisplayedName,
"planName":
inputPlanDisplayedName.replace(' ', '_'),
'chipType':
json_data.get('chipType', ''),
'usePreBeadfind':
toBoolean(json_data['usePreBeadfind'], False),
'usePostBeadfind':
toBoolean(json_data['usePostBeadfind'], False),
'flows':
json_data.get('flows', None),
'autoAnalyze':
True,
'preAnalysis':
True,
'runType':
json_data['runType'],
'library':
json_data.get('library', ''),
'notes':
noteValue,
'bedfile':
json_data.get('bedfile', ''),
'regionfile':
json_data.get('regionfile', ''),
'variantfrequency':
json_data.get('variantfrequency', ''),
'librarykitname':
json_data.get('librarykitname', ''),
'sequencekitname':
json_data.get('sequencekitname', ''),
'barcodeId':
barcodeIdValue,
'templatingKitName':
json_data.get('templatekitname', ''),
'controlSequencekitname':
json_data.get('controlsequence', ''),
'runMode':
runModeValue,
'isSystem':
toBoolean(json_data['isSystem'], False),
'isReusable':
isReusable,
'isPlanGroup':
isPlanGroupValue,
'sampleDisplayedName':
sample.strip(),
"sample":
sample.strip().replace(' ', '_'),
'username':
request.user.username,
'isFavorite':
toBoolean(json_data.get('isFavorite', 'False'), False),
'barcodedSamples':
barcodedSamples,
'libraryKey':
libraryKeyValue,
'forward3primeadapter':
forward3primeAdapterValue,
'reverselibrarykey':
json_data.get('reverselibrarykey', ''),
'reverse3primeadapter':
json_data.get('reverse3primeAdapter', ''),
'pairedEndLibraryAdapterName':
json_data.get('pairedEndLibraryAdapterName', ''),
'samplePrepKitName':
json_data.get('samplePrepKitName', ''),
'selectedPlugins':
selectedPluginsValue
}
#if we're changing a plan from having 1 sample to say 2 samples, we need to UPDATE 1 plan and CREATE 1 plan!!
try:
if not edit_existing_plan:
planTemplate = PlannedExperiment(**kwargs)
else:
planTemplate = PlannedExperiment.objects.get(pk=planOid)
for key, value in kwargs.items():
setattr(planTemplate, key, value)
edit_existing_plan = False
planTemplate.save()
# Update QCtype thresholds
qcTypes = QCType.objects.all()
for qcType in qcTypes:
qc_threshold = json_data.get(qcType.qcName, '')
if qc_threshold:
# get existing PlannedExperimentQC if any
plannedExpQcs = PlannedExperimentQC.objects.filter(
plannedExperiment=planTemplate.id, qcType=qcType.id)
if len(plannedExpQcs) > 0:
for plannedExpQc in plannedExpQcs:
plannedExpQc.threshold = qc_threshold
plannedExpQc.save()
else:
kwargs = {
'plannedExperiment': planTemplate,
'qcType': qcType,
'threshold': qc_threshold
}
plannedExpQc = PlannedExperimentQC(**kwargs)
plannedExpQc.save()
# add/remove projects
if projectObjList:
#TODO: refactor this logic to simplify using django orm
projectNameList = [project.name for project in projectObjList]
for currentProject in planTemplate.projects.all():
if currentProject.name not in projectNameList:
planTemplate.projects.remove(currentProject)
for projectObj in projectObjList:
planTemplate.projects.add(projectObj)
else:
planTemplate.projects.clear()
except:
transaction.rollback()
logger.exception(format_exc())
return HttpResponse(json.dumps(
{"error": "Internal error while trying to save the plan."}),
mimetype="application/json")
else:
transaction.commit()
return HttpResponse(json.dumps(
{"status": "plan template updated successfully"}),
mimetype="application/json")
def test_get_projects(self):
data = {'projects':[],
'newProjects': []}
# data = simplejson.loads(data)
result = get_projects('ionadmin', data)
self.assertFalse(result, 'list should be empty')