def exp_kwargs(d, folder):
"""Converts the output of `parse_log` to the a dictionary of
keyword arguments needed to create an ``Experiment`` database object. """
identical_fields = ("sample", "library", "cycles", "flows",)
simple_maps = (
("experiment_name", "expName"),
("chiptype", "chipType"),
("chipbarcode", "chipBarcode"),
("user_notes", "notes"),
##("seqbarcode", "seqKitBarcode"),
("autoanalyze", "autoAnalyze"),
("prebeadfind", "usePreBeadfind"),
##("librarykeysequence", "libraryKey"),
("barcodeid", "barcodeId"),
("isReverseRun", "isReverseRun"),
)
full_maps = (
("pgmName", d.get('devicename', extract_rig(folder))),
("log", json.dumps(d, indent=4)),
("expDir", folder),
("unique", folder),
("baselineRun", d.get("runtype") == "STD" or d.get("runtype") == "Standard"),
("date", folder_mtime(folder)),
("storage_options", models.GlobalConfig.objects.all()[0].default_storage_options),
("flowsInOrder", getFlowOrder(d.get("image_map", ""))),
("reverse_primer", d.get('reverse_primer', 'Ion Kit')),
)
derive_attribute_list = ["libraryKey", "reverselibrarykey", "forward3primeadapter",
"reverse3primeadapter", "sequencekitname", "seqKitBarcode",
"sequencekitbarcode", "librarykitname", "librarykitbarcode",
"runMode"]
ret = {}
for f in identical_fields:
ret[f] = d.get(f, '')
for k1, k2 in simple_maps:
ret[k2] = d.get(k1, '')
for k, v in full_maps:
ret[k] = v
for attribute in derive_attribute_list:
ret[attribute] = ''
#N.B. this field is not used
ret['storageHost'] = 'localhost'
# If Flows keyword is defined in explog.txt...
if ret['flows'] != "":
# Cycles should be based on number of flows, not cycles published in log file
# (Use of Cycles is deprecated in any case! We should be able to enter a random number here)
ret['cycles'] = int(int(ret['flows']) / len(ret['flowsInOrder']))
else:
# ...if Flows is not defined in explog.txt: (Very-old-dataset support)
ret['flows'] = len(ret['flowsInOrder']) * int(ret['cycles'])
logger.errors.warn("Flows keyword missing: Calculated Flows is %d" % int(ret['flows']))
if ret['barcodeId'].lower() == 'none':
ret['barcodeId'] = ''
if len(d.get('blocks', [])) > 0:
ret['rawdatastyle'] = 'tiled'
ret['autoAnalyze'] = False
for bs in d['blocks']:
# Hack alert. Watch how explogparser.parse_log munges these strings when detecting which one is the thumbnail entry
# Only thumbnail will have 0,0 as first and second element of the string.
if '0' in bs.split(',')[0] and '0' in bs.split(',')[1]:
continue
if auto_analyze_block(bs):
ret['autoAnalyze'] = True
logger.errors.debug("Block Run. Detected at least one block to auto-run analysis")
break
if ret['autoAnalyze'] is False:
logger.errors.debug("Block Run. auto-run whole chip has not been specified")
else:
ret['rawdatastyle'] = 'single'
planShortId = d.get("planned_run_short_id", '')
#fix [TS-3064] for PGM backward compatibility
if (planShortId is None or len(planShortId) == 0):
planShortId = d.get("pending_run_short_id", '')
selectedPlanGUId = d.get("planned_run_guid", '')
logger.errors.debug("...planShortId=%s; selectedPlanGUId=%s" % (planShortId, selectedPlanGUId))
print 'crawler: plannedRunShortId=', planShortId
print 'crawler: plannedRunGUId=', selectedPlanGUId
sequencingKitName = d.get("seqkitname", '')
if sequencingKitName != "NOT_SCANNED":
ret['sequencekitname'] = sequencingKitName
#in rundb_experiment, there are 2 attributes for sequencingKitBarcode!!
sequencingKitBarcode = d.get("seqkitpart", '')
if sequencingKitBarcode != "NOT_SCANNED":
ret['seqKitBarcode'] = sequencingKitBarcode
ret['sequencekitbarcode'] = sequencingKitBarcode
libraryKitName = d.get('libkit', '')
if libraryKitName != "NOT_SCANNED":
ret['librarykitname'] = libraryKitName
libraryKitBarcode = d.get("libbarcode", '')
if libraryKitBarcode != "NOT_SCANNED":
ret['librarykitbarcode'] = libraryKitBarcode
##note: if PGM is running the old version, there is no isReverseRun in explog.txt.
isReverseRun = d.get("isreverserun", '')
if isReverseRun == "Yes":
ret['isReverseRun'] = True
ret['reverselibrarykey'] = ''
ret["runMode"] = "pe"
logger.errors.warn("PAIRED-END is NO LONGER SUPPORTED. Skipping experiment %s" % ret['expName'])
return ret, False
#Rules for applying the library key overrides:
#1) If plan is used and library key is specified, use that value
#2) Otherwise, if user has specified one on PGM's advanced page
# Validation required:
# Why: It could be left-over from a previous run and is not compatible with current run)
# How: It has to be pre-defined in db and is in the direction of the the new run.
# What: If it passes validation, use it
#3) Otherwise, use system default for that direction
#4) If plan is NOT used, and user has specified one in PGM's advanced page, do validation as above
#5) If it passes validation, use it
#6) Otherwise, use system default for that direction as defined in db
#7) If the system default somehow has no value, we'll use the library key from
# PGM's advanced setup page
isPlanFound = False
planObj = None
if selectedPlanGUId:
try:
planObj = models.PlannedExperiment.objects.get(planGUID=selectedPlanGUId)
isPlanFound = True
ret["runMode"] = planObj.runMode
expName = d.get("experiment_name", '')
#fix TS-4714: fail-safe measure, mark the plan executed if instrument somehow does not mark it as executed
if (not planObj.planExecuted):
logger.errors.warn("REPAIR: marking plan %s as executed for experiment %s" % (planObj.planGUID, expName))
planObj.planExecuted = True
planObj.expName = expName
planObj.save()
except models.PlannedExperiment.DoesNotExist:
logger.errors.warn("No plan with GUId %s found in database " % selectedPlanGUId)
except models.PlannedExperiment.MultipleObjectsReturned:
logger.errors.warn("Multiple plan with GUId %s found in database " % selectedPlanGUId)
else:
if (planShortId and len(planShortId) > 0):
try:
#PGM should have set the plan as executed already
#note: if instrument does not include plan GUID for the plan and does not mark the plan as executed,
#crawler will not do any repair (to mark a plan as executed) and actually indexError will likely happen
planObj = models.PlannedExperiment.objects.filter(planShortID=planShortId, planExecuted=True).order_by("-date")[0]
isPlanFound = True
ret["runMode"] = planObj.runMode
planObj.expName = d.get("experiment_name", '')
planObj.save()
logger.errors.debug("...planShortId=%s is for runMode=%s; reverse=%s" % (planShortId, planObj.runMode, planObj.isReverseRun))
except IndexError:
logger.errors.warn("No plan with short id %s found in database " % planShortId)
if not isPlanFound:
#if user does not use a plan for the run, fetch the system default plan template, and clone it for this run
isNeedSystemDefaultTemplate = False
experiment = None
#if we have already saved the experiment to db, and is using the cloned system default plan, explog will
#not know that
try:
experiment = models.Experiment.objects.get(expName=d.get("experiment_name", ''))
logger.errors.debug("crawler experiment already exists!! name=%s" % (d.get("experiment_name", '')))
except models.Experiment.DoesNotExist:
logger.errors.warn("expName: %s not yet in database and may need a sys default plan" % d.get("experiment_name", ''))
#fix TS-4713 if a system default has somehow been cloned for this run, don't bother to clone again
try:
sysDefaultClones = models.PlannedExperiment.objects.filter(expName=d.get("experiment_name", ''))
if sysDefaultClones:
#logger.errors.debug("SKIP cloning system default plan for %s since one already exists " % (d.get("experiment_name", '')))
isNeedSystemDefaultTemplate = False
else:
isNeedSystemDefaultTemplate = True
except:
logger.errors.warn(traceback.format_exc())
isNeedSystemDefaultTemplate = False
except models.Experiment.MultipleObjectsReturned:
#this should not happen since instrument assign uniques run name. But if it happens, don't bother to apply the
#system default plan template to the experiment
logger.errors.warn("multiple expName: %s found in database" % d.get("experiment_name", ''))
isNeedSystemDefaultTemplate = False
if isNeedSystemDefaultTemplate is True:
try:
explogChipType = ret['chipType'][:3]
systemDefaultPlanTemplate = models.PlannedExperiment.objects.filter(isReusable=True, isSystem=True, isSystemDefault=True, chipType = None).order_by("-date")[0]
try:
if explogChipType:
chipSystemDefaultPlanTemplate = models.PlannedExperiment.objects.filter(isReusable=True, isSystem=True, isSystemDefault=True, chipType=explogChipType).order_by("-date")[0]
systemDefaultPlanTemplate = chipSystemDefaultPlanTemplate
logger.errors.debug("Chip-specific system default plan template found in database for chip=%s; experiment=%s" % (explogChipType, d.get("experiment_name", '')))
except IndexError:
logger.errors.debug("No chip-specific system default plan template found in database for chip=%s; experiment=%s. Going to use generic system default template instead." % (explogChipType, d.get("experiment_name", '')))
planObj = copy.copy(systemDefaultPlanTemplate)
planObj.pk = None
planObj.planGUID = None
planObj.planShortID = None
planObj.isReusable = False
planObj.isSystem = False
planObj.isSystemDefault = False
#fix TS-4664: include experiment name to system default clone
expName = d.get("experiment_name", '')
planObj.planName = "CopyOfSystemDefault_" + expName
planObj.expName = expName
planObj.planExecuted = True
if not planObj.chipType:
planObj.chipType = ""
planObj.save()
#clone the qc thresholds as well
qcValues = systemDefaultPlanTemplate.plannedexperimentqc_set.all()
for qcValue in qcValues:
qcObj = copy.copy(qcValue)
qcObj.pk = None
qcObj.plannedExperiment = planObj
qcObj.save()
logger.errors.info("crawler AFTER SAVING SYSTEM DEFAULT CLONE %s for experiment=%s;" % (planObj.planName, expName))
isPlanFound = True
ret["runMode"] = planObj.runMode
except IndexError:
logger.errors.warn("No system default plan template found in database ")
except:
logger.errors.warn(traceback.format_exc())
logger.errors.warn("Error in trying to use system default plan template for experiment=%s" % (d.get("experiment_name", '')))
# planObj is initialized as None, which is an acceptable foreign key value
# for Experiment.plan; however, by this point, we have either found a plan
# or created one from the default plan template, so there should be a plan
# and in all three cases, None, found plan, and default plan, we're ready
# to commit the plan object to the Experiment.plan foreign key relationship.
ret["plan"] = planObj
#if PGM is running the old version, there is no isReverseRun in explog.txt. Check the plan if used
if not isReverseRun:
if isPlanFound:
if planObj.isReverseRun:
isReverseRun = "Yes"
else:
isReverseRun = "No"
if isReverseRun == "Yes":
ret['isReverseRun'] = True
ret['reverselibrarykey'] = ''
ret["runMode"] = "pe"
logger.errors.warn("PAIRED-END is NO LONGER SUPPORTED. Skipping experiment %s" % ret['expName'])
return ret, False
else:
ret['isReverseRun'] = False
ret['libraryKey'] = ''
if isPlanFound is False:
ret["runMode"] = "single"
defaultPairedEndForward3primeAdapter = None
try:
#Note: In v3.0, plan has concept of "runMode".
#TS-4524: allow crawler to be more tolerant, especially PE 3' adapter is only used for PE run
defaultPairedEndForward3primeAdapter = models.ThreePrimeadapter.objects.get(direction="Forward", name__iexact="Ion Paired End Fwd")
except models.ThreePrimeadapter.DoesNotExist:
logger.errors.warn("No default pairedEnd forward 3' adapter in database for experiment %s" % ret['expName'])
except models.ThreePrimeadapter.MultipleObjectsReturned:
logger.errors.warn("Multiple default pairedEnd forward 3' adapters found in database for experiment %s" % ret['expName'])
try:
#NOTE: In v2.2, there is no way to tell if a run (aka an experiment) is part of a paired-end forward run or not
defaultForwardLibraryKey = models.LibraryKey.objects.get(direction='Forward', isDefault=True)
defaultForward3primeAdapter = models.ThreePrimeadapter.objects.get(direction='Forward', isDefault=True)
validatedPgmLibraryKey = None
dbPgmLibraryKey = None
pgmLibraryKey = d.get("librarykeysequence", '')
#logger.errors.debug("...pgmLibraryKey is %s " % pgmLibraryKey)
hasPassed = False
if pgmLibraryKey is None or len(pgmLibraryKey) == 0:
#logger.errors.debug("...pgmLibraryKey not specified. ")
hasPassed = False
else:
dbPgmLibraryKeys = models.LibraryKey.objects.filter(sequence=pgmLibraryKey)
if dbPgmLibraryKeys:
for dbKey in dbPgmLibraryKeys:
if dbKey.direction == "Forward":
#logger.errors.debug("...pgmLibraryKey %s has been validated for forward run" % pgmLibraryKey)
validatedPgmLibraryKey = dbKey
hasPassed = True
break
else:
hasPassed = False
#set default in case plan is not used or not found in db
if hasPassed:
#logger.errors.debug("...Default for forward run. Use PGM library key=%s " % validatedPgmLibraryKey.sequence)
ret['libraryKey'] = validatedPgmLibraryKey.sequence
else:
#logger.errors.debug("...Default for forward run. Use default library key=%s " % defaultForwardLibraryKey.sequence)
ret['libraryKey'] = defaultForwardLibraryKey.sequence
ret['forward3primeadapter'] = defaultForward3primeAdapter.sequence
if isPlanFound:
#logger.errors.debug("...FORWARD plan is FOUND for planShortId=%s " % planShortId)
if planObj.libraryKey:
#logger.errors.debug("...Plan used for forward run. Use plan library key=%s " % planObj.libraryKey)
ret['libraryKey'] = planObj.libraryKey
else:
if hasPassed:
#logger.errors.debug("...Plan used for forward run. Use PGM library key=%s " % validatedPgmLibraryKey.sequence)
ret['libraryKey'] = validatedPgmLibraryKey.sequence
else:
#logger.errors.debug("...Plan used for forward run. Use default library key=%s " % defaultForwardLibraryKey.sequence)
ret['libraryKey'] = defaultForwardLibraryKey.sequence
if planObj.forward3primeadapter:
ret['forward3primeadapter'] = planObj.forward3primeadapter
else:
if (planObj.runMode == "pe"):
if defaultPairedEndForward3primeAdapter:
ret['forward3primeadapter'] = defaultPairedEndForward3primeAdapter.sequence
else:
ret['forward3primeadapter'] = ""
else:
ret['forward3primeadapter'] = defaultForward3primeAdapter.sequence
else:
if hasPassed:
#logger.errors.debug("...Plan used but not on db for forward run. Use PGM library key=%s " % validatedPgmLibraryKey.sequence)
ret['libraryKey'] = validatedPgmLibraryKey.sequence
else:
#logger.errors.debug("...Plan used but not on db for forward run. Use default library key=%s " % defaultForwardLibraryKey.sequence)
ret['libraryKey'] = defaultForwardLibraryKey.sequence
ret['forward3primeadapter'] = defaultForward3primeAdapter.sequence
if ret['libraryKey'] is None or ret['libraryKey'] == "":
#logger.errors.debug("...A library key cannot be determined for this FORWARD run Use PGM default. ")
ret['libraryKey'] = d.get("librarykeysequence", '')
except models.LibraryKey.DoesNotExist:
logger.errors.warn("No default forward library key in database for experiment %s" % ret['expName'])
return ret, False
except models.LibraryKey.MultipleObjectsReturned:
logger.errors.warn("Multiple default forward library keys found in database for experiment %s" % ret['expName'])
return ret, False
except models.ThreePrimeadapter.DoesNotExist:
logger.errors.warn("No default forward 3' adapter in database for experiment %s" % ret['expName'])
return ret, False
except models.ThreePrimeadapter.MultipleObjectsReturned:
logger.errors.warn("Multiple default forward 3' adapters found in database for experiment %s" % ret['expName'])
return ret, False
except:
logger.errors.warn("Experiment %s" % ret['expName'])
logger.errors.warn(traceback.format_exc())
return ret, False
# Limit input sizes to defined field widths in models.py
ret['notes'] = ret['notes'][:1024]
ret['expDir'] = ret['expDir'][:512]
ret['expName'] = ret['expName'][:128]
ret['pgmName'] = ret['pgmName'][:64]
ret['unique'] = ret['unique'][:512]
ret['storage_options'] = ret['storage_options'][:200]
# ret['project'] = ret['project'][:64]
ret['sample'] = ret['sample'][:64]
ret['library'] = ret['library'][:64]
ret['chipBarcode'] = ret['chipBarcode'][:64]
ret['seqKitBarcode'] = ret['seqKitBarcode'][:64]
ret['chipType'] = ret['chipType'][:32]
ret['flowsInOrder'] = ret['flowsInOrder'][:512]
ret['libraryKey'] = ret['libraryKey'][:64]
ret['barcodeId'] = ret['barcodeId'][:128]
ret['reverse_primer'] = ret['reverse_primer'][:128]
ret['reverselibrarykey'] = ret['reverselibrarykey'][:64]
ret['reverse3primeadapter'] = ret['reverse3primeadapter'][:512]
ret['forward3primeadapter'] = ret['forward3primeadapter'][:512]
ret['sequencekitbarcode'] = ret['sequencekitbarcode'][:512]
ret['librarykitbarcode'] = ret['librarykitbarcode'][:512]
ret['sequencekitname'] = ret['sequencekitname'][:512]
ret['sequencekitbarcode'] = ret['sequencekitbarcode'][:512]
ret['librarykitname'] = ret['librarykitname'][:512]
ret['librarykitbarcode'] = ret['librarykitbarcode'][:512]
ret['runMode'] = ret['runMode'][:64]
logger.errors.debug("For experiment %s" % ret['expName'])
logger.errors.debug("...Ready to save run: isReverseRun=%s;" % ret['isReverseRun'])
logger.errors.debug("...Ready to save run: libraryKey=%s;" % ret['libraryKey'])
logger.errors.debug("...Ready to save run: forward3primeadapter=%s;" % ret['forward3primeadapter'])
logger.errors.debug("...Ready to save run: reverselibrarykey=%s;" % ret['reverselibrarykey'])
logger.errors.debug("...Ready to save run: reverse3primeadapter=%s;" % ret['reverse3primeadapter'])
return ret, True
def exp_kwargs(d, folder, logobj):
"""Converts the output of `parse_log` to the a dictionary of
keyword arguments needed to create an ``Experiment`` database object.
"""
identical_fields = ("sample", "cycles", "flows", "project",)
simple_maps = (
("experiment_name", "expName"),
("chipbarcode", "chipBarcode"),
("user_notes", "notes"),
##("seqbarcode", "seqKitBarcode"),
("autoanalyze", "autoAnalyze"),
("prebeadfind", "usePreBeadfind"),
("librarykeysequence", "libraryKey"),
("barcodeid", "barcodeKitName"),
("isReverseRun", "isReverseRun"),
("library", "reference"),
)
chiptype = d.get('chiptype','')
chipversion = d.get('chipversion','')
if chipversion:
chiptype = chipversion
if chiptype.startswith('1.10'):
chiptype = 'P1.1.17'
elif chiptype.startswith('1.20'):
chiptype = 'P1.2.18'
full_maps = (
("chipType", chiptype),
("pgmName", d.get('devicename', extract_rig(folder))),
("log", json.dumps(d, indent=4)),
("expDir", folder),
("unique", folder),
("baselineRun", d.get("runtype") == "STD" or d.get("runtype") == "Standard"),
("date", explog_time(d.get("start_time", ""), folder)),
("storage_options", models.GlobalConfig.objects.all()[0].default_storage_options),
("flowsInOrder", getFlowOrder(d.get("image_map", ""))),
("reverse_primer", d.get('reverse_primer', 'Ion Kit')),
)
derive_attribute_list = ["sequencekitname", "seqKitBarcode", "sequencekitbarcode",
"libraryKitName", "libraryKitBarcode"]
ret = {}
for f in identical_fields:
ret[f] = d.get(f, '')
for k1, k2 in simple_maps:
ret[k2] = d.get(k1, '')
for k, v in full_maps:
ret[k] = v
for attribute in derive_attribute_list:
ret[attribute] = ''
#N.B. this field is not used
ret['storageHost'] = 'localhost'
# If Flows keyword is defined in explog.txt...
if ret['flows'] != "":
# Cycles should be based on number of flows, not cycles published in log file
# (Use of Cycles is deprecated in any case! We should be able to enter a random number here)
ret['cycles'] = int(int(ret['flows']) / len(ret['flowsInOrder']))
else:
# ...if Flows is not defined in explog.txt: (Very-old-dataset support)
ret['flows'] = len(ret['flowsInOrder']) * int(ret['cycles'])
logobj.warn("Flows keyword missing: Calculated Flows is %d" % int(ret['flows']))
if ret['barcodeKitName'].lower() == 'none':
ret['barcodeKitName'] = ''
if len(d.get('blocks', [])) > 0:
ret['rawdatastyle'] = 'tiled'
ret['autoAnalyze'] = False
for bs in d['blocks']:
# Hack alert. Watch how explogparser.parse_log munges these strings when detecting which one is the thumbnail entry
# Only thumbnail will have 0,0 as first and second element of the string.
if '0' in bs.split(',')[0] and '0' in bs.split(',')[1]:
continue
if auto_analyze_block(bs, logobj):
ret['autoAnalyze'] = True
logobj.debug("Block Run. Detected at least one block to auto-run analysis")
break
if ret['autoAnalyze'] is False:
logobj.debug("Block Run. auto-run whole chip has not been specified")
else:
ret['rawdatastyle'] = 'single'
sequencingKitName = d.get("seqkitname", '')
#do not replace plan's seqKit info if explog has blank seqkitname
if sequencingKitName and sequencingKitName != "NOT_SCANNED":
ret['sequencekitname'] = sequencingKitName
#in rundb_experiment, there are 2 attributes for sequencingKitBarcode!!
sequencingKitBarcode = d.get("seqkitpart", '')
if sequencingKitBarcode and sequencingKitBarcode != "NOT_SCANNED":
ret['seqKitBarcode'] = sequencingKitBarcode
ret['sequencekitbarcode'] = sequencingKitBarcode
libraryKitBarcode = d.get("libbarcode", '')
if libraryKitBarcode and libraryKitBarcode != "NOT_SCANNED":
ret['libraryKitBarcode'] = libraryKitBarcode
libraryKitName = d.get('libkit', '')
if libraryKitName and libraryKitName != "NOT_SCANNED":
ret['libraryKitName'] = libraryKitName
##note: if PGM is running the old version, there is no isReverseRun in explog.txt.
isReverseRun = d.get("isreverserun", '')
if isReverseRun == "Yes":
ret['isReverseRun'] = True
else:
ret['isReverseRun'] = False
#instrument could have blank runType or be absent all together in explog
runType = d.get('runtype', "")
if not runType:
runType = "GENS"
ret['runType'] = runType
# Limit input sizes to defined field widths in models.py
ret['notes'] = ret['notes'][:1024]
ret['expDir'] = ret['expDir'][:512]
ret['expName'] = ret['expName'][:128]
ret['pgmName'] = ret['pgmName'][:64]
ret['unique'] = ret['unique'][:512]
ret['storage_options'] = ret['storage_options'][:200]
ret['project'] = ret['project'][:64]
ret['sample'] = ret['sample'][:64]
ret['reference'] = ret['reference'][:64]
ret['chipBarcode'] = ret['chipBarcode'][:64]
ret['seqKitBarcode'] = ret['seqKitBarcode'][:64]
ret['chipType'] = ret['chipType'][:32]
ret['flowsInOrder'] = ret['flowsInOrder'][:512]
ret['libraryKey'] = ret['libraryKey'][:64]
ret['barcodeKitName'] = ret['barcodeKitName'][:128]
ret['reverse_primer'] = ret['reverse_primer'][:128]
ret['sequencekitname'] = ret['sequencekitname'][:512]
ret['sequencekitbarcode'] = ret['sequencekitbarcode'][:512]
ret['libraryKitName'] = ret['libraryKitName'][:512]
ret['libraryKitBarcode'] = ret['libraryKitBarcode'][:512]
ret['runType'] = ret['runType'][:512]
return ret
def exp_kwargs(d, folder):
"""Converts the output of `parse_log` to the a dictionary of
keyword arguments needed to create an ``Experiment`` database object.
"""
identical_fields = (
"sample",
"cycles",
"flows",
"project",
)
simple_maps = (
("experiment_name", "expName"),
("chiptype", "chipType"),
("chipbarcode", "chipBarcode"),
("user_notes", "notes"),
##("seqbarcode", "seqKitBarcode"),
("autoanalyze", "autoAnalyze"),
("prebeadfind", "usePreBeadfind"),
("librarykeysequence", "libraryKey"),
("barcodeid", "barcodeKitName"),
("isReverseRun", "isReverseRun"),
("library", "reference"),
)
full_maps = (
("pgmName", d.get('devicename', extract_rig(folder))),
("log", json.dumps(d, indent=4)),
("expDir", folder),
("unique", folder),
("baselineRun", d.get("runtype") == "STD"
or d.get("runtype") == "Standard"),
("date", explog_time(d.get("start_time", ""), folder)),
("storage_options",
models.GlobalConfig.objects.all()[0].default_storage_options),
("flowsInOrder", getFlowOrder(d.get("image_map", ""))),
("reverse_primer", d.get('reverse_primer', 'Ion Kit')),
)
derive_attribute_list = [
"sequencekitname", "seqKitBarcode", "sequencekitbarcode",
"libraryKitName", "libraryKitBarcode"
]
ret = {}
for f in identical_fields:
ret[f] = d.get(f, '')
for k1, k2 in simple_maps:
ret[k2] = d.get(k1, '')
for k, v in full_maps:
ret[k] = v
for attribute in derive_attribute_list:
ret[attribute] = ''
#N.B. this field is not used
ret['storageHost'] = 'localhost'
# If Flows keyword is defined in explog.txt...
if ret['flows'] != "":
# Cycles should be based on number of flows, not cycles published in log file
# (Use of Cycles is deprecated in any case! We should be able to enter a random number here)
ret['cycles'] = int(int(ret['flows']) / len(ret['flowsInOrder']))
else:
# ...if Flows is not defined in explog.txt: (Very-old-dataset support)
ret['flows'] = len(ret['flowsInOrder']) * int(ret['cycles'])
logger.errors.warn("Flows keyword missing: Calculated Flows is %d" %
int(ret['flows']))
if ret['barcodeKitName'].lower() == 'none':
ret['barcodeKitName'] = ''
if len(d.get('blocks', [])) > 0:
ret['rawdatastyle'] = 'tiled'
ret['autoAnalyze'] = False
for bs in d['blocks']:
# Hack alert. Watch how explogparser.parse_log munges these strings when detecting which one is the thumbnail entry
# Only thumbnail will have 0,0 as first and second element of the string.
if '0' in bs.split(',')[0] and '0' in bs.split(',')[1]:
continue
if auto_analyze_block(bs):
ret['autoAnalyze'] = True
logger.errors.debug(
"Block Run. Detected at least one block to auto-run analysis"
)
break
if ret['autoAnalyze'] is False:
logger.errors.debug(
"Block Run. auto-run whole chip has not been specified")
else:
ret['rawdatastyle'] = 'single'
sequencingKitName = d.get("seqkitname", '')
#do not replace plan's seqKit info if explog has blank seqkitname
if sequencingKitName and sequencingKitName != "NOT_SCANNED":
ret['sequencekitname'] = sequencingKitName
#in rundb_experiment, there are 2 attributes for sequencingKitBarcode!!
sequencingKitBarcode = d.get("seqkitpart", '')
if sequencingKitBarcode and sequencingKitBarcode != "NOT_SCANNED":
ret['seqKitBarcode'] = sequencingKitBarcode
ret['sequencekitbarcode'] = sequencingKitBarcode
libraryKitBarcode = d.get("libbarcode", '')
if libraryKitBarcode and libraryKitBarcode != "NOT_SCANNED":
ret['libraryKitBarcode'] = libraryKitBarcode
libraryKitName = d.get('libkit', '')
if libraryKitName and libraryKitName != "NOT_SCANNED":
ret['libraryKitName'] = libraryKitName
##note: if PGM is running the old version, there is no isReverseRun in explog.txt.
isReverseRun = d.get("isreverserun", '')
if isReverseRun == "Yes":
ret['isReverseRun'] = True
else:
ret['isReverseRun'] = False
#instrument could have blank runType or be absent all together in explog
runType = d.get('runtype', "")
if not runType:
runType = "GENS"
ret['runType'] = runType
# Limit input sizes to defined field widths in models.py
ret['notes'] = ret['notes'][:1024]
ret['expDir'] = ret['expDir'][:512]
ret['expName'] = ret['expName'][:128]
ret['pgmName'] = ret['pgmName'][:64]
ret['unique'] = ret['unique'][:512]
ret['storage_options'] = ret['storage_options'][:200]
ret['project'] = ret['project'][:64]
ret['sample'] = ret['sample'][:64]
ret['reference'] = ret['reference'][:64]
ret['chipBarcode'] = ret['chipBarcode'][:64]
ret['seqKitBarcode'] = ret['seqKitBarcode'][:64]
ret['chipType'] = ret['chipType'][:32]
ret['flowsInOrder'] = ret['flowsInOrder'][:512]
ret['libraryKey'] = ret['libraryKey'][:64]
ret['barcodeKitName'] = ret['barcodeKitName'][:128]
ret['reverse_primer'] = ret['reverse_primer'][:128]
ret['sequencekitname'] = ret['sequencekitname'][:512]
ret['sequencekitbarcode'] = ret['sequencekitbarcode'][:512]
ret['libraryKitName'] = ret['libraryKitName'][:512]
ret['libraryKitBarcode'] = ret['libraryKitBarcode'][:512]
ret['runType'] = ret['runType'][:512]
return ret
