def buildPaneKeyLookup ():
global paneKeyLookup, refKey
#
# Get the reference key for the J-Number.
#
refKey = loadlib.verifyReference(jNumber, 0, None)
#
# Get all the figure labels and associated image pane keys for the
# reference.
#
results = db.sql('''select i.figureLabel, ip._ImagePane_key
from IMG_Image i, IMG_ImagePane ip
where i._Image_key = ip._Image_key
and i._ImageType_key = %d
and i._Refs_key = %d''' % (FULLSIZE_IMAGE_TYPE_KEY, refKey), 'auto')
for r in results:
figureLabel = r['figureLabel']
paneKey = r['_ImagePane_key']
paneKeyLookup[figureLabel] = paneKey
print 'paneKeyLookup[' + figureLabel + '] = ' + str(paneKey)
return
def init ():
global createdByKey, refKey, accKey
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFile)
#
# Get the created by key for the user.
#
createdByKey = loadlib.verifyUser(createdBy, 0, None)
#
# Get the reference key for the J-Number.
#
refKey = loadlib.verifyReference(jNumber, 0, None)
#
# Get the next available accession key.
#
results = db.sql('select max(_Accession_key) + 1 as maxKey from ACC_Accession', 'auto')
accKey = results[0]['maxKey']
return
def processFile():
# requires:
#
# effects:
# Reads input file
# Verifies and Processes each line in the input file
#
# returns:
# nothing
#
global refAssocKey
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
accID = tokens[0]
jnum = tokens[1]
refAssocType = tokens[2]
createdBy = tokens[3]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
objectKey = loadlib.verifyObject(accID, mgiTypeKey, None, lineNum, errorFile)
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
refAssocTypeKey = verifyRefAssocType(refAssocType, lineNum)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if objectKey == 0 or \
referenceKey == 0 or \
refAssocTypeKey == 0 or \
createdByKey == 0:
# set error flag to true
error = 1
# if errors, continue to next record
if error:
continue
# if no errors, process the marker
# could move to verifyDuplicate routine
key = '%s:%s:%s' % (objectKey, referenceKey, refAssocTypeKey)
if refDict.has_key(key):
errorFile.write('Duplicate (%d) %s\n' % (lineNum, line))
continue
refFile.write('%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (refAssocKey, referenceKey, objectKey, mgiTypeKey, refAssocTypeKey, createdByKey, createdByKey, loaddate, loaddate))
refAssocKey = refAssocKey + 1
def processFile():
global probeKey, refKey, aliasKey, accKey, mgiKey
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
name = tokens[0]
jnum = tokens[1]
parentID = tokens[2]
sourceName = tokens[3]
organism = tokens[4]
strain = tokens[5]
tissue = tokens[6]
gender = tokens[7]
cellLine = tokens[8]
age = tokens[9]
vectorType = tokens[10]
segmentType = tokens[11]
regionCovered = tokens[12]
insertSite = tokens[13]
insertSize = tokens[14]
markerIDs = string.split(tokens[15], '|')
relationship = tokens[16]
sequenceIDs = tokens[17]
aliasList = string.split(tokens[18], '|')
notes = tokens[19]
rawnotes = tokens[20]
createdBy = tokens[21]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
isParent = 0
isSource = 0
parentProbeKey = '';
sourceKey = 0
if parentID != '':
isParent = 1
if sourceName != '':
isSource = 1
if not isParent and not isSource:
organismKey = sourceloadlib.verifyOrganism(organism, lineNum, errorFile)
strainKey = sourceloadlib.verifyStrain(strain, lineNum, errorFile)
tissueKey = sourceloadlib.verifyTissue(tissue, lineNum, errorFile)
genderKey = sourceloadlib.verifyGender(gender, lineNum, errorFile)
cellLineKey = sourceloadlib.verifyCellLine(cellLine, lineNum, errorFile)
vectorKey = sourceloadlib.verifyVectorType(vectorType, lineNum, errorFile)
segmentTypeKey = sourceloadlib.verifySegmentType(segmentType, lineNum, errorFile)
sourceKey = sourceloadlib.verifySource(segmentTypeKey, \
vectorKey, organismKey, strainKey, \
tissueKey, genderKey, cellLineKey, age, lineNum, errorFile)
if organismKey == 0 or strainKey == 0 or tissueKey == 0 or \
genderKey == 0 or cellLineKey == 0 or vectorKey == 0 or \
segmentTypeKey == 0 or sourceKey == 0:
errorFile.write('%s, %s, %s, %s, %s, %s, %s, %s\n' % (segmentType, vectorType, organism, strain, tissue, gender, cellLine, age))
error = 1
elif not isParent and isSource:
vectorKey = sourceloadlib.verifyVectorType(vectorType, lineNum, errorFile)
segmentTypeKey = sourceloadlib.verifySegmentType(segmentType, lineNum, errorFile)
sourceKey = sourceloadlib.verifyLibrary(sourceName, lineNum, errorFile)
if vectorKey == 0 or segmentTypeKey == 0 or sourceKey == 0:
error = 1
# parent from = yes, source given = yes or no (ignored)
else:
parentProbeKey, sourceKey = verifyParentProbe(parentID, lineNum, errorFile)
vectorKey = sourceloadlib.verifyVectorType(vectorType, lineNum, errorFile)
segmentTypeKey = sourceloadlib.verifySegmentType(segmentType, lineNum, errorFile)
if parentProbeKey == 0 or sourceKey == 0 or vectorKey == 0 or segmentTypeKey == 0:
error = 1
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if referenceKey == 0:
errorFile.write('Invalid Reference: %s\n' % (jnum))
error = 1
if createdByKey == 0:
errorFile.write('Invalid Creator: %s\n\n' % (createdBy))
error = 1
# marker IDs
markerList = []
for markerID in markerIDs:
markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile)
if len(markerID) > 0 and markerKey == 0:
errorFile.write('Invalid Marker: %s, %s\n' % (name, markerID))
error = 1
elif len(markerID) > 0:
markerList.append(markerKey)
# sequence IDs
seqAccDict = {}
for seqID in string.split(sequenceIDs, '|'):
if len(seqID) > 0:
[logicalDB, acc] = string.split(seqID, ':')
logicalDBKey = loadlib.verifyLogicalDB(logicalDB, lineNum, errorFile)
if logicalDBKey > 0:
seqAccDict[acc] = logicalDBKey
# if errors, continue to next record
if error:
continue
# if no errors, process the probe
probeFile.write('%d\t%s\t%s\t%s\t%s\t%s\t\t\t%s\t%s\t%s\t\t%s\t%s\t%s\t%s\n' \
% (probeKey, name, parentProbeKey, sourceKey, vectorKey, segmentTypeKey, mgi_utils.prvalue(regionCovered), \
mgi_utils.prvalue(insertSite), mgi_utils.prvalue(insertSize), createdByKey, createdByKey, loaddate, loaddate))
for markerKey in markerList:
if markerList.count(markerKey) == 1:
markerFile.write('%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\n' \
% (probeKey, markerKey, referenceKey, relationship, createdByKey, createdByKey, loaddate, loaddate))
else:
errorFile.write('Invalid Marker Duplicate: %s, %s\n' % (name, markerID))
refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' \
% (refKey, probeKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate))
# aliases
for alias in aliasList:
if len(alias) == 0:
continue
aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
% (aliasKey, refKey, alias, createdByKey, createdByKey, loaddate, loaddate))
aliasKey = aliasKey + 1
# MGI Accession ID for the marker
accFile.write('%s\t%s%d\t%s\t%s\t1\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \
% (accKey, mgiPrefix, mgiKey, mgiPrefix, mgiKey, probeKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate))
# Print out a new text file and attach the new MGI Probe IDs as the last field
newProbeFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%d\n' \
% (name, jnum, \
mgi_utils.prvalue(sourceName), \
organism, \
mgi_utils.prvalue(strain), \
mgi_utils.prvalue(tissue), \
mgi_utils.prvalue(gender), \
mgi_utils.prvalue(cellLine), \
mgi_utils.prvalue(age), \
mgi_utils.prvalue(vectorType), \
mgi_utils.prvalue(segmentType), \
mgi_utils.prvalue(regionCovered) + \
mgi_utils.prvalue(insertSite), \
mgi_utils.prvalue(insertSize), \
string.join(markerIDs, '|'), \
relationship, \
mgi_utils.prvalue(sequenceIDs), \
string.join(aliasList, '|'), \
mgi_utils.prvalue(notes), \
createdBy, mgiPrefix, mgiKey))
# Print out a raw note file
if len(rawnotes) > 0:
rawNoteFile.write('%s%d\t%s\n' % (mgiPrefix, mgiKey, rawnotes))
# Notes
if len(notes) > 0:
noteFile.write('%s\t%s\t%s\t%s\n' % (probeKey, notes, loaddate, loaddate))
accKey = accKey + 1
mgiKey = mgiKey + 1
# sequence accession ids
for acc in seqAccDict.keys():
prefixPart, numericPart = accessionlib.split_accnum(acc)
accFile.write('%s\t%s\t%s\t%s\t%s\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \
% (accKey, acc, prefixPart, numericPart, seqAccDict[acc], probeKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate))
accRefFile.write('%s\t%s\t%s\t%s\t%s\t%s\n' \
% (accKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate))
accKey = accKey + 1
refKey = refKey + 1
probeKey = probeKey + 1
# end of "for line in inputFile.readlines():"
#
# Update the AccessionMax value
#
if not DEBUG:
db.sql('select * from ACC_setMax (%d)' % (lineNum), None)
def processFile():
global refKey, aliasKey
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
probeID = probeName = tokens[0]
jnum = tokens[1]
aliasList = string.split(tokens[2], '|')
createdBy = tokens[3]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
if probeID.find('MGI:') >= 0:
probeKey = loadlib.verifyProbe(probeID, lineNum, errorFile)
else:
probeKey, probeID = verifyProbe(probeName, lineNum, errorFile)
probeReferenceKey = verifyProbeReference(probeID, jnum, lineNum,
errorFile)
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if probeKey == 0:
errorFile.write('Invalid Probe: %s\n' % (probeID))
error = 1
if referenceKey == 0:
errorFile.write('Invalid Reference: %s\n' % (jnum))
error = 1
#if probeReferenceKey == 0:
# errorFile.write('Invalid Probe Reference: %s, %s\n' % (probeID, jnum))
# error = 1
if createdByKey == 0:
errorFile.write('Invalid Creator: %s\n\n' % (createdBy))
error = 1
# if errors, continue to next record
if error:
continue
# if no errors, process
# create a new probe-reference key if one does not already exist
# else use the existing probe-reference key
if probeReferenceKey == 0:
refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' \
% (refKey, probeKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate))
aliasrefKey = refKey
refKey = refKey + 1
else:
#errorFile.write('Probe/Reference Already Exists: %s\n' % (tokens))
aliasrefKey = probeReferenceKey
# aliases
for alias in aliasList:
if len(alias) == 0:
continue
aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
% (aliasKey, aliasrefKey, alias, createdByKey, createdByKey, loaddate, loaddate))
aliasKey = aliasKey + 1
def processFile():
'''
# requires:
#
# effects:
# Reads input file
# Verifies and Processes each line in the input file
#
# returns:
# nothing
#
'''
global referenceKey
global exptDict, seqExptDict
lineNum = 0
note = ''
# For each line in the input file
inputFile = open(inputFileName, 'r')
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '|')
try:
markerID = tokens[0]
chromosome = tokens[1]
updateChr = tokens[2]
band = tokens[3]
assay = tokens[4]
description = tokens[5]
jnum = tokens[6]
createdBy = tokens[7]
except:
# if it's not a valid line, assume it's the note
note = line
continue
# exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
markerKey, markerSymbol = verifyMarker(markerID, lineNum)
assayKey = verifyAssay(assay)
referenceKey = loadlib.verifyReference(jnum, 0, errorFile)
createdByKey = loadlib.verifyUser(createdBy, 0, errorFile)
error = not verifyChromosome(chromosome, lineNum)
if markerKey == 0 or \
assayKey == 0 or \
referenceKey == 0 or \
createdByKey == 0:
# set error flag to true
error = 1
# if errors, continue to next record
if error:
continue
# if no errors, process
# run once...needs the reference
if lineNum == 1:
createExperimentMaster()
# determine experiment key for this chromosome
# if it doesn't exist, create it
if not exptDict.has_key(chromosome):
createExperimentBCP(chromosome)
if not exptDict.has_key(chromosome):
errorFile.write('Cannot Find Experiment Key For Chromosome (%d): %s\n' % (lineNum, chromosome))
chrExptKey = 0
else:
chrExptKey = exptDict[chromosome]
# if errors, continue to next record
if chrExptKey == 0:
continue
# add marker to experiment marker file
bcpWrite(exptMarkerFile, \
[chrExptKey, \
markerKey, \
alleleKey, \
assayKey, \
seqExptDict[chrExptKey], \
markerSymbol, \
description, \
matrixData, \
loaddate, loaddate])
# increment marker sequence number for the experiment
seqExptDict[chrExptKey] = seqExptDict[chrExptKey] + 1
# end of "for line in inputFile.readlines():"
if len(note) > 0:
bcpWrite(noteFile, [referenceKey, note, loaddate, loaddate])
def processFile():
global refKey, aliasKey
global execProbeSQL
global execAssaySQL
global execRefSQL
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
fromID = tokens[0]
name = tokens[1]
toID = tokens[2]
jnum = tokens[3]
createdBy = tokens[4]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
fromKey = loadlib.verifyObject(fromID, mgiTypeKey, None, lineNum, errorFile)
toKey = loadlib.verifyObject(toID, mgiTypeKey, None, lineNum, errorFile)
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if fromKey == 0:
errorFile.write('Invalid Probe "From": %s\n' % (fromID))
error = 1
if toKey == 0:
errorFile.write('Invalid Probe "To": %s\n' % (toID))
error = 1
if referenceKey == 0:
errorFile.write('Invalid Reference: %s\n' % (jnum))
error = 1
if createdByKey == 0:
errorFile.write('Invalid Creator: %s\n\n' % (createdBy))
error = 1
# check that all genes are the same
checkGenesSQL = '''
select f.*
from PRB_Marker f, PRB_Marker t, GXD_ProbePrep p, GXD_Assay a
where f._Probe_key = %s
and t._Probe_key = %s
and p._Probe_key = %s
and p._ProbePrep_key = a._ProbePrep_key
and f._Marker_key = t._Marker_key
and f._Marker_key = a._Marker_key
''' % (fromKey, toKey, fromKey)
checkGenes = db.sql(checkGenesSQL, 'auto')
if len(checkGenes) == 0:
errorFile.write('Gene of GenePaint, Eurexpress and Assay are not the same: %s, %s\n' % (fromID, toID))
error = 1
# check that the J: is on at least one Assay
checkJAssaySQL = '''
select a.*
from GXD_ProbePrep p, GXD_Assay a
where p._Probe_key = %s
and p._ProbePrep_key = a._ProbePrep_key
and a._Refs_key = %s
''' % (fromKey, referenceKey)
checkJAssay = db.sql(checkJAssaySQL, 'auto')
if len(checkJAssay) == 0:
errorFile.write('J: is not on any Assays attached to the probe: %s\n' % (fromID))
error = 1
# if errors, continue to next record
if error:
continue
# add alias using fromID name (from) to toID
refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' \
% (refKey, toKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate))
aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
% (aliasKey, refKey, name, createdByKey, createdByKey, loaddate, loaddate))
refKey = refKey + 1
aliasKey = aliasKey + 1
# move assay information from fromID to toID
execAssaySQL.append(updateAssaySQL % (toKey, fromKey))
# move fromID (from) references to toID
execRefSQL.append(updateRefSQL % (toKey, fromKey, referenceKey))
# delete fromID (from)
execProbeSQL.append(deleteProbeSQL % (fromKey))
def processFile():
global refKey, aliasKey, execSQL
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
probeID = tokens[0]
markerIDs = string.split(tokens[1], '|')
jnum = tokens[2]
relationship = tokens[3]
aliasList = string.split(tokens[4], '|')
createdBy = tokens[5]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
probeKey = loadlib.verifyProbe(probeID, lineNum, errorFile)
refsKey = loadlib.verifyReference(jnum, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if probeKey == 0:
errorFile.write('Invalid Probe: %s\n' % (probeID))
error = 1
if refsKey == 0:
errorFile.write('Invalid Reference: %s\n' % (jnum))
error = 1
if createdByKey == 0:
errorFile.write('Invalid Creator: %s\n\n' % (createdBy))
error = 1
results = db.sql('''select _Reference_key from PRB_Reference
where _Probe_key = %s
and _Refs_key = %s
''' % (probeKey, refsKey), 'auto')
referenceKey = results[0]['_Reference_key']
if referenceKey == 0:
errorFile.write('Invalid Probe/Reference: %s\n' % (jnum))
error = 1
# marker IDs
markerList = []
for markerID in markerIDs:
if markerID == 'none':
break
markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile)
if markerKey == 0:
errorFile.write('Invalid Marker: %s\n' % (markerID))
error = 1
else:
markerList.append(markerKey)
# if errors, continue to next record
if error:
continue
# if no errors, process
for markerKey in markerList:
if markerList.count(markerKey) == 1:
markerFile.write('%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\n' \
% (probeKey, markerKey, refsKey, relationship, createdByKey, createdByKey, loaddate, loaddate))
execSQL.append(deleteSQL % (probeKey, markerKey))
else:
errorFile.write('Invalid Marker Duplicate: %s\n' % (markerID))
if referenceKey > 0:
refKey = referenceKey
else:
refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' \
% (refKey, probeKey, refsKey, createdByKey, createdByKey, loaddate, loaddate))
# aliases
for alias in aliasList:
aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
% (aliasKey, refKey, alias, createdByKey, createdByKey, loaddate, loaddate))
aliasKey = aliasKey + 1
# only used if referenceKey == 0
refKey = refKey + 1
def processFile():
global refKey, aliasKey, execSQL
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
probeID = tokens[0]
markerIDs = string.split(tokens[1], '|')
jnum = tokens[2]
relationship = tokens[3]
aliasList = string.split(tokens[4], '|')
createdBy = tokens[5]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
probeKey = loadlib.verifyProbe(probeID, lineNum, errorFile)
refsKey = loadlib.verifyReference(jnum, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if probeKey == 0:
errorFile.write('Invalid Probe: %s\n' % (probeID))
error = 1
if refsKey == 0:
errorFile.write('Invalid Reference: %s\n' % (jnum))
error = 1
if createdByKey == 0:
errorFile.write('Invalid Creator: %s\n\n' % (createdBy))
error = 1
results = db.sql(
'''select _Reference_key from PRB_Reference
where _Probe_key = %s
and _Refs_key = %s
''' % (probeKey, refsKey), 'auto')
referenceKey = results[0]['_Reference_key']
if referenceKey == 0:
errorFile.write('Invalid Probe/Reference: %s\n' % (jnum))
error = 1
# marker IDs
markerList = []
for markerID in markerIDs:
if markerID == 'none':
break
markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile)
if markerKey == 0:
errorFile.write('Invalid Marker: %s\n' % (markerID))
error = 1
else:
markerList.append(markerKey)
# if errors, continue to next record
if error:
continue
# if no errors, process
for markerKey in markerList:
if markerList.count(markerKey) == 1:
markerFile.write('%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\n' \
% (probeKey, markerKey, refsKey, relationship, createdByKey, createdByKey, loaddate, loaddate))
execSQL.append(deleteSQL % (probeKey, markerKey))
else:
errorFile.write('Invalid Marker Duplicate: %s\n' % (markerID))
if referenceKey > 0:
refKey = referenceKey
else:
refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' \
% (refKey, probeKey, refsKey, createdByKey, createdByKey, loaddate, loaddate))
# aliases
for alias in aliasList:
aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
% (aliasKey, refKey, alias, createdByKey, createdByKey, loaddate, loaddate))
aliasKey = aliasKey + 1
# only used if referenceKey == 0
refKey = refKey + 1
def init():
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes local DBMS parameters
# 3. Initializes global file descriptors/file names
# 4. Initializes global keys
#
# returns:
#
global diagFileName, errorFileName, synFileName
global inputFile, diagFile, errorFile, synFile
global mgiTypeKey, createdByKey, referenceKey
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
head, tail = os.path.split(inputFileName)
diagFileName = logDir + '/' + tail + '.diagnostics'
errorFileName = logDir + '/' + tail + '.error'
synFileName = 'MGI_Synonym.bcp'
print inputFileName
print logDir
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
synFile = open(outputDir + '/' + synFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % synFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
diagFile.write('Object Type: %s\n' % (mgiType))
diagFile.write('Input File: %s\n' % (inputFileName))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
mgiTypeKey = loadlib.verifyMGIType(mgiType, 0, errorFile)
createdByKey = loadlib.verifyUser(createdBy, 0, errorFile)
# if reference is J:0, then no reference is given
if jnum == 'J:0':
referenceKey = ''
else:
referenceKey = loadlib.verifyReference(jnum, 0, errorFile)
# exit if we can't resolve mgiType, createdBy or jnum
if mgiTypeKey == 0 or \
createdByKey == 0 or \
referenceKey == 0:
exit(1)
if mode == 'reload':
print 'mode is: %s, deleting synonyms' % mode
sys.stdout.flush()
db.sql('delete from MGI_Synonym ' + \
'where _MGIType_key = %d ' % (mgiTypeKey) + \
'and _CreatedBy_key = %d ' % (createdByKey), None)
def processFile():
global refKey, aliasKey
global execProbeSQL
global execAssaySQL
global execRefSQL
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
fromID = tokens[0]
name = tokens[1]
toID = tokens[2]
jnum = tokens[3]
createdBy = tokens[4]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
fromKey = loadlib.verifyObject(fromID, mgiTypeKey, None, lineNum,
errorFile)
toKey = loadlib.verifyObject(toID, mgiTypeKey, None, lineNum,
errorFile)
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if fromKey == 0:
errorFile.write('Invalid Probe "From": %s\n' % (fromID))
error = 1
if toKey == 0:
errorFile.write('Invalid Probe "To": %s\n' % (toID))
error = 1
if referenceKey == 0:
errorFile.write('Invalid Reference: %s\n' % (jnum))
error = 1
if createdByKey == 0:
errorFile.write('Invalid Creator: %s\n\n' % (createdBy))
error = 1
# check that all genes are the same
checkGenesSQL = '''
select f.*
from PRB_Marker f, PRB_Marker t, GXD_ProbePrep p, GXD_Assay a
where f._Probe_key = %s
and t._Probe_key = %s
and p._Probe_key = %s
and p._ProbePrep_key = a._ProbePrep_key
and f._Marker_key = t._Marker_key
and f._Marker_key = a._Marker_key
''' % (fromKey, toKey, fromKey)
checkGenes = db.sql(checkGenesSQL, 'auto')
if len(checkGenes) == 0:
errorFile.write(
'Gene of GenePaint, Eurexpress and Assay are not the same: %s, %s\n'
% (fromID, toID))
error = 1
# check that the J: is on at least one Assay
checkJAssaySQL = '''
select a.*
from GXD_ProbePrep p, GXD_Assay a
where p._Probe_key = %s
and p._ProbePrep_key = a._ProbePrep_key
and a._Refs_key = %s
''' % (fromKey, referenceKey)
checkJAssay = db.sql(checkJAssaySQL, 'auto')
if len(checkJAssay) == 0:
errorFile.write(
'J: is not on any Assays attached to the probe: %s\n' %
(fromID))
error = 1
# if errors, continue to next record
if error:
continue
# add alias using fromID name (from) to toID
refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' \
% (refKey, toKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate))
aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
% (aliasKey, refKey, name, createdByKey, createdByKey, loaddate, loaddate))
refKey = refKey + 1
aliasKey = aliasKey + 1
# move assay information from fromID to toID
execAssaySQL.append(updateAssaySQL % (toKey, fromKey))
# move fromID (from) references to toID
execRefSQL.append(updateRefSQL % (toKey, fromKey, referenceKey))
# delete fromID (from)
execProbeSQL.append(deleteProbeSQL % (fromKey))
def processFile():
'''
# requires:
#
# effects:
# Reads input file
# Verifies and Processes each line in the input file
#
# returns:
# nothing
#
'''
global referenceKey
global exptDict, seqExptDict
lineNum = 0
note = ''
# For each line in the input file
inputFile = open(inputFileName, 'r')
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = str.split(line[:-1], '|')
try:
mappingKey = tokens[0]
markerID = tokens[1]
chromosome = tokens[2]
updateChr = tokens[3]
band = tokens[4]
assay = tokens[5]
description = tokens[6]
jnum = tokens[7]
createdBy = tokens[8]
except:
# if it's not a valid line, assume it's the note
note = line
continue
# exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
markerKey, markerSymbol = verifyMarker(markerID, lineNum)
assayKey = verifyAssay(assay)
referenceKey = loadlib.verifyReference(jnum, 0, errorFile)
createdByKey = loadlib.verifyUser(createdBy, 0, errorFile)
error = not verifyChromosome(chromosome, lineNum)
if markerKey == 0 or \
assayKey == 0 or \
referenceKey == 0 or \
createdByKey == 0:
# set error flag to true
error = 1
# if errors, continue to next record
if error:
continue
# if no errors, process
# run once...needs the reference
if lineNum == 1:
createExperimentMaster()
# determine experiment key for this chromosome
# if it doesn't exist, create it
if chromosome not in exptDict:
createExperimentBCP(chromosome)
if chromosome not in exptDict:
errorFile.write(
'Cannot Find Experiment Key For Chromosome (%d): %s\n' %
(lineNum, chromosome))
chrExptKey = 0
else:
chrExptKey = exptDict[chromosome]
# if errors, continue to next record
if chrExptKey == 0:
continue
# add marker to experiment marker file
bcpWrite(exptMarkerFile, \
[mappingKey, \
chrExptKey, \
markerKey, \
alleleKey, \
assayKey, \
seqExptDict[chrExptKey], \
description, \
matrixData, \
loaddate, loaddate])
# increment marker sequence number for the experiment
seqExptDict[chrExptKey] = seqExptDict[chrExptKey] + 1
# end of "for line in inputFile.readlines():"
if len(note) > 0:
bcpWrite(noteFile, [referenceKey, note, loaddate, loaddate])
def processFile():
global alleleKey, refAssocKey, accKey, noteKey, mgiKey, annotKey, mutationKey
global alleleLookup
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = line[:-1].split('\t')
#print line
try:
markerID = tokens[0]
symbol = tokens[1]
name = tokens[2]
alleleStatus = tokens[3]
alleleType = tokens[4]
alleleSubtypes = tokens[5]
collectionKey = tokens[6]
germLine = tokens[7]
references = tokens[8]
strainOfOrigin = tokens[9]
mutantCellLine = tokens[10]
molecularNotes = tokens[11]
driverNotes = tokens[12]
ikmcNotes = tokens[13]
mutations = tokens[14]
inheritanceMode = tokens[15]
isMixed = tokens[16]
isExtinct = tokens[17]
createdBy = tokens[18]
createMCL = tokens[19]
createNote = tokens[20]
setStatus = tokens[21]
existingAlleleID = tokens[22]
ikmcSymbol = tokens[23]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
# creator
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if createdByKey == 0:
continue
# processing for IKMC-only
if len(createMCL) > 0 or len(createNote) > 0 or len(setStatus) > 0:
processFileIKMC(createMCL, createNote, setStatus, \
symbol, ikmcSymbol, mutantCellLine, ikmcNotes, \
createdByKey, existingAlleleID)
continue
# marker key
markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile)
# hard-coded
# _vocab_key = 73 (Marker-Allele Association Status)
# _term_key = 4268545 (Curated)
markerStatusKey = 4268545
# _vocab_key = 37 (Allele Status)
alleleStatusKey = loadlib.verifyTerm('', 37, alleleStatus, lineNum,
errorFile)
# _vocab_key = 38 (Allele Type)
alleleTypeKey = loadlib.verifyTerm('', 38, alleleType, lineNum,
errorFile)
# _vocab_key = 61 (Allele Transmission)
germLineKey = loadlib.verifyTerm('', 61, germLine, lineNum, errorFile)
# _vocab_key = 36 (Allele Molecular Mutation)
allMutations = mutations.split('|')
# _vocab_key = 35 (Allele Status)
inheritanceModeKey = loadlib.verifyTerm('', 35, inheritanceMode,
lineNum, errorFile)
# strains
strainOfOriginKey = sourceloadlib.verifyStrain(strainOfOrigin, lineNum,
errorFile)
# reference
refKey = loadlib.verifyReference(jnum, lineNum, errorFile)
# if errors, continue to next record
# errors are stored (via loadlib) in the .error log
if markerKey == 0 \
or markerStatusKey == 0 \
or alleleStatusKey == 0 \
or alleleTypeKey == 0 \
or germLineKey == 0 \
or allMutations == 0 \
or inheritanceModeKey == 0 \
or strainOfOriginKey == 0 \
or refKey == 0 \
or createdByKey == 0:
continue
# if no errors, process the allele
# not specified/testing
#collectionKey = 11025586
# allele (master)
alleleFile.write('%d|%s|%s|%s|%s|%s|%s|%s|%s|%s|0|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (alleleKey, markerKey, strainOfOriginKey, inheritanceModeKey, alleleTypeKey, \
alleleStatusKey, germLineKey, collectionKey, symbol, name, \
isExtinct, isMixed, refKey, markerStatusKey, \
createdByKey, createdByKey, createdByKey, loaddate, loaddate, loaddate))
# molecular mutation
for mutation in allMutations:
mutationTermKey = loadlib.verifyTerm('', 36, mutation, lineNum,
errorFile)
mutationFile.write('%s|%s|%s|%s|%s\n' \
% (mutationKey, alleleKey, mutationTermKey, loaddate, loaddate))
mutationKey = mutationKey + 1
#
# allele references
#
allReferences = references.split('||')
for reference in allReferences:
refType, refID = reference.split('|')
refKey = loadlib.verifyReference(refID, lineNum, errorFile)
if refType == 'Original':
refAssocTypeKey = 1011
elif refType == 'Transmission':
refAssocTypeKey = 1023
elif refType == 'Molecular':
refAssocTypeKey = 1012
refFile.write('%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (refAssocKey, refKey, alleleKey, mgiTypeKey, refAssocTypeKey, \
createdByKey, createdByKey, loaddate, loaddate))
refAssocKey = refAssocKey + 1
#
# allele subtypes
#
allSubtypes = alleleSubtypes.split('|')
for s in allSubtypes:
# _vocab_key = 93 (Allele Subtype)
alleleSubtypeKey = loadlib.verifyTerm('', 93, s, lineNum,
errorFile)
annotFile.write('%s|%s|%s|%s|%s|%s|%s\n' \
% (annotKey, annotTypeKey, alleleKey, alleleSubtypeKey, \
qualifierKey, loaddate, loaddate))
annotKey = annotKey + 1
#
# mutant cell line
#
if len(mutantCellLine) > 0:
addMutantCellLine(alleleKey, mutantCellLine, createdByKey)
# MGI Accession ID for the allelearker
accFile.write('%s|%s%d|%s|%s|1|%d|%d|0|1|%s|%s|%s|%s\n' \
% (accKey, mgiPrefix, mgiKey, mgiPrefix, mgiKey, alleleKey, mgiTypeKey, \
createdByKey, createdByKey, loaddate, loaddate))
# storing data in MGI_Note
# molecular notes
mgiNoteSeqNum = 1
if len(molecularNotes) > 0:
noteFile.write('%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (noteKey, alleleKey, mgiNoteObjectKey, mgiMolecularNoteTypeKey, \
molecularNotes, createdByKey, createdByKey, loaddate, loaddate))
noteKey = noteKey + 1
# driver notes
# TR12662/MGI_Relationship._Category_key = 1006
# removed noteFile code
# place hodler for MGI_Relationship code
# the IKMC is the only product using this and IKMC does not add any driver note
#mgiNoteSeqNum = 1
#if len(driverNotes) > 0:
# ikmc notes
useIKMCnotekey = 0
if len(ikmcNotes) > 0:
noteFile.write('%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (noteKey, alleleKey, mgiNoteObjectKey, mgiIKMCNoteTypeKey, \
ikmcNotes, createdByKey, createdByKey, loaddate, loaddate))
useIKMCnotekey = noteKey
noteKey = noteKey + 1
# Print out a new text file and attach the new MGI Allele IDs as the last field
if createdBy == 'ikmc_alleleload':
newAlleleFile.write('%s\t%s%s\t%s\n' \
% (mgi_utils.prvalue(ikmcNotes), \
mgi_utils.prvalue(mgiPrefix), mgi_utils.prvalue(mgiKey), \
mgi_utils.prvalue(ikmcSymbol)))
else:
newAlleleFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%s\n' \
% (mgi_utils.prvalue(markerID), \
mgi_utils.prvalue(symbol), \
mgi_utils.prvalue(name), \
mgi_utils.prvalue(alleleStatus), \
mgi_utils.prvalue(alleleType), \
mgi_utils.prvalue(alleleSubtype), \
mgi_utils.prvalue(collection), \
mgi_utils.prvalue(germLine), \
mgi_utils.prvalue(references), \
mgi_utils.prvalue(strainOfOrigin), \
mgi_utils.prvalue(mutantCellLine), \
mgi_utils.prvalue(allMutations), \
mgi_utils.prvalue(inheritanceMode), \
mgi_utils.prvalue(isMixed), \
mgi_utils.prvalue(isExtinct), \
mgi_utils.prvalue(refKey), \
mgi_utils.prvalue(markerStatusKey), \
mgi_utils.prvalue(createdBy), \
mgi_utils.prvalue(mgiPrefix), mgi_utils.prvalue(mgiKey)))
# save symbol/alleleKey/ikmc note key
alleleLookup[symbol] = []
alleleLookup[symbol].append(
(alleleKey, useIKMCnotekey, mgiPrefix + str(mgiKey)))
accKey = accKey + 1
mgiKey = mgiKey + 1
alleleKey = alleleKey + 1
# end of "for line in inputFile.readlines():"
#
# Update the AccessionMax value
#
if not DEBUG:
db.sql('select * from ACC_setMax(%d)' % (lineNum), None)
db.commit()
def processFile():
# requires:
#
# effects:
# Reads input file
# Verifies and Processes each line in the input file
#
# returns:
# nothing
#
global refAssocKey
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
accID = tokens[0]
jnum = tokens[1]
refAssocType = tokens[2]
createdBy = tokens[3]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
objectKey = loadlib.verifyObject(accID, mgiTypeKey, None, lineNum,
errorFile)
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
refAssocTypeKey = verifyRefAssocType(refAssocType, lineNum)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if objectKey == 0 or \
referenceKey == 0 or \
refAssocTypeKey == 0 or \
createdByKey == 0:
# set error flag to true
error = 1
# if errors, continue to next record
if error:
continue
# if no errors, process the marker
# could move to verifyDuplicate routine
key = '%s:%s:%s' % (objectKey, referenceKey, refAssocTypeKey)
if refDict.has_key(key):
errorFile.write('Duplicate (%d) %s\n' % (lineNum, line))
continue
refFile.write('%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (refAssocKey, referenceKey, objectKey, mgiTypeKey, refAssocTypeKey, createdByKey, createdByKey, loaddate, loaddate))
refAssocKey = refAssocKey + 1
def processFile():
global primerKey, refKey, aliasKey, accKey, mgiKey
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
markerSymbol = tokens[0] # not used
markerIDs = string.split(tokens[1], '|')
name = tokens[2]
jnum = tokens[3]
regionCovered = tokens[4]
sequence1 = tokens[5]
sequence2 = tokens[6]
productSize = tokens[7]
notes = tokens[8]
sequenceIDs = tokens[9]
aliasList = string.split(tokens[10], '|')
createdBy = tokens[11]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
# marker IDs
markerList = []
for markerID in markerIDs:
markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile)
if len(markerID) > 0 and markerKey == 0:
errorFile.write('Invalid Marker: %s, %s\n' % (name, markerID))
error = 1
elif len(markerID) > 0:
markerList.append(markerKey)
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
# sequence IDs
seqAccList = string.split(sequenceIDs, '|')
# if errors, continue to next record
if error:
continue
# if no errors, process the primer
primerFile.write('%d\t%s\t\t%d\t%d\t%s\t%s\t%s\t%s\t\t\t%s\t%s\t%s\t%s\t%s\n' \
% (primerKey, name, NA, vectorKey, segmentTypeKey, mgi_utils.prvalue(sequence1), \
mgi_utils.prvalue(sequence2), mgi_utils.prvalue(regionCovered), mgi_utils.prvalue(productSize), \
createdByKey, createdByKey, loaddate, loaddate))
for markerKey in markerList:
if markerList.count(markerKey) == 1:
markerFile.write('%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\n' \
% (primerKey, markerKey, referenceKey, relationship, createdByKey, createdByKey, loaddate, loaddate))
else:
errorFile.write('Invalid Marker Duplicate: %s, %s\n' % (name, markerID))
# loaddate))
refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' % (refKey, primerKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate))
# aliases
for alias in aliasList:
if len(alias) == 0:
continue
aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
% (aliasKey, refKey, alias, createdByKey, createdByKey, loaddate, loaddate))
aliasKey = aliasKey + 1
# MGI Accession ID for the marker
accFile.write('%s\t%s%d\t%s\t%s\t1\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \
% (accKey, mgiPrefix, mgiKey, mgiPrefix, mgiKey, primerKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate))
newPrimerFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%d\n' \
% (markerSymbol, string.join(markerIDs, '|'), name, jnum, regionCovered, sequence1, sequence2, productSize, notes, sequenceIDs, createdBy, mgiPrefix, mgiKey))
accKey = accKey + 1
mgiKey = mgiKey + 1
# sequence accession ids
for acc in seqAccList:
if len(acc) == 0:
continue
prefixPart, numericPart = accessionlib.split_accnum(acc)
accFile.write('%s\t%s\t%s\t%s\t%s\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \
% (accKey, acc, prefixPart, numericPart, logicalDBKey, primerKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate))
accRefFile.write('%s\t%s\t%s\t%s\t%s\t%s\n' \
% (accKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate))
accKey = accKey + 1
# notes
if len(notes) > 0:
noteFile.write('%s|1\t%s\t%s\t%s\n' \
% (primerKey, notes, loaddate, loaddate))
refKey = refKey + 1
primerKey = primerKey + 1
# end of "for line in inputFile.readlines():"
#
# Update the AccessionMax value
#
if not DEBUG:
db.sql('select * from ACC_setMax (%d)' % (lineNum), None)
def processImageFile():
global imageKey, accKey, mgiKey
global imagePix
global referenceKey
lineNum = 0
# For each line in the input file
for line in inImageFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
jnum = tokens[0]
fullsizeKey = tokens[1]
imageClass = tokens[2]
pixID = tokens[3]
xdim = tokens[4]
ydim = tokens[5]
figureLabel = tokens[6]
copyrightNote = tokens[7]
imageNote = tokens[8]
imageInfo = tokens[9]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
imageClassKey = loadlib.verifyTerm('', imageVocabClassKey, imageClass, lineNum, errorFile)
if imageClassKey == 0:
error = 1
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
if referenceKey == 0:
error = 1
# if errors, continue to next record
if error:
continue
# if no errors, process
imageTypeKey = FSimageTypeKey
outImageFile.write(str(imageKey) + TAB + \
str(gxdMgiTypeKey) + TAB + \
str(imageClassKey) + TAB + \
str(imageTypeKey) + TAB + \
str(referenceKey) + TAB + \
TAB + \
xdim + TAB + \
ydim + TAB + \
figureLabel + TAB + \
str(createdByKey) + TAB + \
str(createdByKey) + TAB + \
loaddate + TAB + loaddate + CRT)
# MGI Accession ID for the image
mgiAccID = mgiPrefix + str(mgiKey)
outAccFile.write(str(accKey) + TAB + \
mgiPrefix + str(mgiKey) + TAB + \
mgiPrefix + TAB + \
str(mgiKey) + TAB + \
accLogicalDBKey + TAB + \
str(imageKey) + TAB + \
imageMgiTypeKey + TAB + \
accPrivate + TAB + \
accPreferred + TAB + \
str(createdByKey) + TAB + \
str(createdByKey) + TAB + \
loaddate + TAB + loaddate + CRT)
accKey = accKey + 1
mgiKey = mgiKey + 1
if pixID.find('GUDMAP') < 0 and len(pixID) > 0:
outAccFile.write(str(accKey) + TAB + \
pixPrefix + str(pixID) + TAB + \
pixPrefix + TAB + \
pixID + TAB + \
pixLogicalDBKey + TAB + \
str(imageKey) + TAB + \
imageMgiTypeKey + TAB + \
pixPrivate + TAB + \
accPreferred + TAB + \
str(createdByKey) + TAB + \
str(createdByKey) + TAB + \
loaddate + TAB + loaddate + CRT)
accKey = accKey + 1
if len(imageInfo) > 0:
imageLogicalDBKey, imageID = imageInfo.split('|')
outAccFile.write(str(accKey) + TAB + \
imageID + TAB + \
imageID + TAB + \
TAB + \
imageLogicalDBKey + TAB + \
str(imageKey) + TAB + \
imageMgiTypeKey + TAB + \
accPrivate + TAB + \
accPreferred + TAB + \
str(createdByKey) + TAB + \
str(createdByKey) + TAB + \
loaddate + TAB + loaddate + CRT)
accKey = accKey + 1
# Copyrights
if len(copyrightNote) > 0:
outCopyrightFile.write(mgiAccID + TAB + copyrightNote + CRT)
# Notes
if len(imageNote) > 0:
outCaptionFile.write(mgiAccID + TAB + imageNote + CRT)
imagePix[pixID] = imageKey
imageKey = imageKey + 1
# end of "for line in inImageFile.readlines():"
return lineNum
def processFile():
# Purpose: processes input file
# Returns: nothing
# Assumes: nothing
# Effects: nothing
# Throws: nothing
global libraryName, libraryID, libraryKey, logicalDBKey
global segmentTypeKey, vectorTypeKey, organismKey, referenceKey, strainKey, tissueKey
global age, ageMin, ageMax, genderKey, cellLineKey, createdByKey
global strainNS, tissueNS, genderNS, cellLineNS, ageNS
lineNum = 0
# retrieve next available primary key for Library record
results = db.sql(
'select maxKey = max(_Source_key) + 1 from %s' % (libraryTable),
'auto')
newlibraryKey = results[0]['maxKey']
strainNS = sourceloadlib.verifyStrain(NS, 0, None)
tissueNS = sourceloadlib.verifyTissue(NS, 0, None)
genderNS = sourceloadlib.verifyGender(NS, 0, None)
cellLineNS = sourceloadlib.verifyCellLine(NS, 0, None)
ageNS = NS
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
try:
[libraryName, \
logicalDB, \
libraryID, \
segmentType, \
vectorType, \
organism, \
strain, \
tissue, \
age, \
gender, \
cellLine, \
jnum, \
note, \
cloneCollections, \
createdBy] = string.split(line[:-1], TAB)
except:
exit(1, 'Invalid Line (line: %d): %s\n' % (lineNum, line))
continue
libraryKey = sourceloadlib.verifyLibrary(libraryName, lineNum)
if len(logicalDB) > 0:
logicalDBKey = loadlib.verifyLogicalDB(logicalDB, lineNum,
errorFile)
else:
logicalDBKey = 0
if libraryKey == 0 and len(libraryID) > 0:
libraryKey = sourceloadlib.verifyLibraryID(libraryID, logicalDBKey,
lineNum, errorFile)
segmentTypeKey = sourceloadlib.verifySegmentType(
segmentType, lineNum, errorFile)
vectorTypeKey = sourceloadlib.verifyVectorType(vectorType, lineNum,
errorFile)
strainKey = sourceloadlib.verifyStrain(strain, lineNum, errorFile)
tissueKey = sourceloadlib.verifyTissue(tissue, lineNum, errorFile)
genderKey = sourceloadlib.verifyGender(gender, lineNum, errorFile)
cellLineKey = sourceloadlib.verifyCellLine(cellLine, lineNum,
errorFile)
ageMin, ageMax = sourceloadlib.verifyAge(age, lineNum, errorFile)
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if segmentTypeKey == 0 or \
vectorTypeKey == 0 or \
strainKey == 0 or \
tissueKey == 0 or \
genderKey == 0 or \
cellLineKey == 0 or \
organismKey == 0 or \
referenceKey == 0 or \
createdByKey == 0 or \
ageMin is None:
# set error flag to true
error = 1
# print str(segmentTypeKey)
# print str(vectorTypeKey)
# print str(strainKey)
# print str(tissueKey)
# print str(genderKey)
# print str(cellLineKey)
# print str(organismKey)
# print str(referenceKey)
# print str(createdByKey)
# print str(ageMin)
errorFile.write('Errors: %s\n' % (libraryName))
# if errors, continue to next record
if error:
continue
# if no errors, continue processing
# process new library
if libraryKey == 0:
libraryKey = newlibraryKey
addLibrary()
# increment primary keys
newlibraryKey = newlibraryKey + 1
# else, process existing library
else:
updateLibrary()
addCloneCollections(cloneCollections)
return
def processFile():
global execSQL
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
probeID = tokens[0]
markerIDs = string.split(tokens[1], '|')
jnum = tokens[2]
relationship = tokens[3]
createdBy = tokens[4]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
probeKey = loadlib.verifyProbe(probeID, lineNum, errorFile)
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if probeKey == 0:
errorFile.write('Invalid Probe: %s\n' % (probeID))
error = 1
if referenceKey == 0:
errorFile.write('Invalid Reference: %s\n' % (jnum))
error = 1
if createdByKey == 0:
errorFile.write('Invalid Creator: %s\n\n' % (createdBy))
error = 1
# marker IDs
markerList = []
for markerID in markerIDs:
markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile)
if markerKey == 0:
errorFile.write('Invalid Marker: %s, %s\n' % (name, markerID))
error = 1
else:
markerList.append(markerKey)
# if errors, continue to next record
if error:
continue
# if no errors, process
for markerKey in markerList:
if markerList.count(markerKey) == 1:
markerFile.write('%s|%s|%d|%s|%s|%s|%s|%s\n' \
% (probeKey, markerKey, referenceKey, relationship, createdByKey, createdByKey, loaddate, loaddate))
execSQL.append(deleteSQL % (probeKey, markerKey))
else:
errorFile.write('Invalid Marker Duplicate: %s, %s\n' %
(name, markerID))
def processFile():
# Purpose: Read the input file, resolve values to keys. Create bcp files
# Returns: 1 if error, else 0
# Assumes: file descriptors have been initialized
# Effects: exits if the line does not have 15 columns
# Throws: Nothing
global alleleKey, refAssocKey, accKey, noteKey, mgiKey, annotKey
global alleleLookup, alleleMutationKey
lineNum = 0
# For each line in the input file
for line in fpInputFile.readlines():
error = 0
lineNum = lineNum + 1
print('%s: %s' % (lineNum, line))
# Split the line into tokens
tokens = line[:-1].split('\t')
try:
markerID = tokens[0]
markerSymbol = tokens[1]
mutationType = tokens[2] # IMPC allele type
description = tokens[3]
colonyID = tokens[4]
strainOfOrigin = tokens[5]
alleleSymbol = tokens[6]
alleleName = tokens[7]
inheritanceMode = tokens[8]
alleleType = tokens[9] # IMPC allele class
alleleSubType = tokens[10]
alleleStatus = tokens[11]
transmission = tokens[12]
collection = tokens[13]
jNum = tokens[14]
createdBy = tokens[15]
except:
print('exiting with invalid line')
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
print('validating data and getting keys')
# marker key
markerKey = loadlib.verifyMarker(markerID, lineNum, fpErrorFile)
# _vocab_key = 36 (Allele Molecular Mutation)
mutationList = str.split(mutationType, ';')
if len(mutationList) > 1:
print('mutationList: %s' % mutationList)
mutationKeyList = []
for m in mutationList:
mutationKey = loadlib.verifyTerm('', 36, m, lineNum, fpErrorFile)
if mutationKey != 0:
mutationKeyList.append(mutationKey)
if len(mutationKeyList) > 1:
print('mutationKeyList: %s' % mutationKeyList)
# strains
strainOfOriginKey = sourceloadlib.verifyStrain(strainOfOrigin, lineNum,
fpErrorFile)
# _vocab_key = 35 (Allele Inheritance Mode)
inheritanceModeKey = loadlib.verifyTerm('', 35, inheritanceMode,
lineNum, fpErrorFile)
# _vocab_key = 38 (Allele Type)
alleleTypeKey = loadlib.verifyTerm('', 38, alleleType, lineNum,
fpErrorFile)
# _vocab_key = 93 (Allele Subtype)
subTypeList = str.split(alleleSubType, ';')
if len(subTypeList) > 1:
print('subTypeList: %s' % subTypeList)
subTypeKeyList = []
for s in subTypeList:
if s != '': # if we have a subtype, get it's key
subTypeKey = loadlib.verifyTerm('', 93, s, lineNum,
fpErrorFile)
if subTypeKey != 0:
subTypeKeyList.append(subTypeKey)
if len(subTypeKeyList) > 1:
print('subTypeKeyList: %s' % subTypeKeyList)
# _vocab_key = 37 (Allele Status)
alleleStatusKey = loadlib.verifyTerm('', 37, alleleStatus, lineNum,
fpErrorFile)
# _vocab_key = 61 (Allele Transmission)
transmissionKey = loadlib.verifyTerm('', 61, transmission, lineNum,
fpErrorFile)
# _vocab_key = 92
collectionKey = loadlib.verifyTerm('', 92, collection, lineNum,
fpErrorFile)
# _vocab_key = 73 (Marker-Allele Association Status)
# _term_key = 4268545 (Curated)
markerStatusKey = 4268545
# reference
refKey = loadlib.verifyReference(jNum, lineNum, fpErrorFile)
# creator
createdByKey = loadlib.verifyUser(createdBy, lineNum, fpErrorFile)
if createdByKey == 0:
continue
print('checking for missing data')
# if errors, continue to next record
# errors are stored (via loadlib) in the .error log
if markerKey == 0 \
or mutationKeyList == [] \
or strainOfOriginKey == 0 \
or inheritanceModeKey == 0 \
or alleleTypeKey == 0 \
or alleleStatusKey == 0 \
or transmissionKey == 0 \
or collectionKey == 0 \
or refKey == 0 \
or createdByKey == 0:
print('missing data, skipping this line')
continue
# if no errors, process the allele
print('writing to allele file')
# allele (isWildType = 0)
fpAlleleFile.write('%d|%s|%s|%s|%s|%s|%s|%s|%s|%s|0|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (alleleKey, markerKey, strainOfOriginKey, inheritanceModeKey, alleleTypeKey, \
alleleStatusKey, transmissionKey, collectionKey, alleleSymbol, alleleName, \
isExtinct, isMixed, refKey, markerStatusKey, \
createdByKey, createdByKey, createdByKey, loaddate, loaddate, loaddate))
# molecular mutation
for mutationKey in mutationKeyList:
fpMutationFile.write('%s|%s|%s|%s|%s\n' \
% (alleleMutationKey, alleleKey, mutationKey, loaddate, loaddate))
alleleMutationKey += 1
# reference associations
# Original
fpRefFile.write('%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (refAssocKey, refKey, alleleKey, mgiTypeKey, origRefTypeKey, \
createdByKey, createdByKey, loaddate, loaddate))
refAssocKey = refAssocKey + 1
# Molecular
fpRefFile.write('%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (refAssocKey, refKey, alleleKey, mgiTypeKey, molRefTypeKey, \
createdByKey, createdByKey, loaddate, loaddate))
refAssocKey = refAssocKey + 1
# allele subtype
for subTypeKey in subTypeKeyList:
fpAnnotFile.write('%s|%s|%s|%s|%s|%s|%s\n' \
% (annotKey, annotTypeKey, alleleKey, subTypeKey, \
qualifierKey, loaddate, loaddate))
annotKey = annotKey + 1
# MGI Accession ID for the allele
alleleID = '%s%s' % (mgiPrefix, mgiKey)
fpAccFile.write('%s|%s|%s|%s|1|%d|%d|0|1|%s|%s|%s|%s\n' \
% (accKey, alleleID, mgiPrefix, mgiKey, alleleKey, mgiTypeKey, \
createdByKey, createdByKey, loaddate, loaddate))
# storing data in MGI_Note
# molecular note
fpNoteFile.write('%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (noteKey, alleleKey, mgiTypeKey, molecularNoteTypeKey, description,\
createdByKey, createdByKey, loaddate, loaddate))
noteKey = noteKey + 1
# colony ID note
fpNoteFile.write('%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (noteKey, alleleKey, mgiTypeKey, colonyIdNoteTypeKey, colonyID, \
createdByKey, createdByKey, loaddate, loaddate))
noteKey = noteKey + 1
# Print out a new text file and attach the new MGI Allele IDs
# as the last field
fpNewAlleleRptFile.write('%s\t%s\t%s\t%s\t%s\t%s\n' \
% (mgi_utils.prvalue(alleleID), \
mgi_utils.prvalue(alleleSymbol), \
mgi_utils.prvalue(alleleName), \
mgi_utils.prvalue(markerID), \
mgi_utils.prvalue(markerSymbol), \
mgi_utils.prvalue(colonyID)))
accKey = accKey + 1
mgiKey = mgiKey + 1
alleleKey = alleleKey + 1
#
# Update the AccessionMax value
#
print('DEBUG: %s' % DEBUG)
if DEBUG == 'false':
db.sql('select * from ACC_setMax(%d)' % (lineNum), None)
db.commit()
return 0
def processAssayFile():
global assayAssay, assayKey, accKey, mgiKey
lineNum = 0
# For each line in the input file
for line in inAssayFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], TAB)
try:
assayID = tokens[0]
markerID = tokens[1]
jnum = tokens[2]
assayType = tokens[3]
reporterGene = tokens[4]
note = tokens[5]
createdBy = tokens[6]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile)
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
assayTypeKey = gxdloadlib.verifyAssayType(assayType, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if markerKey == 0 or referenceKey == 0 or assayTypeKey == 0:
# set error flag to true
error = 1
if len(reporterGene) > 0:
reporterGeneKey = gxdloadlib.verifyReporterGene(reporterGene, lineNum, errorFile)
if reporterGeneKey == 0:
error = 1
else:
reporterGeneKey = ''
# if errors, continue to next record
if error:
continue
if assayProbePrep.has_key(assayID):
probePrepKey = assayProbePrep[assayID]
else:
probePrepKey = ''
# if no errors, process
outAssayFile.write(str(assayKey) + TAB + \
str(assayTypeKey) + TAB + \
str(referenceKey) + TAB + \
str(markerKey) + TAB + \
str(probePrepKey) + TAB + \
TAB + \
TAB + \
str(reporterGeneKey) + TAB + \
str(createdByKey) + TAB + \
str(createdByKey) + TAB + \
loaddate + TAB + loaddate + CRT)
if len(note) > 0:
i = 0
while i < len(note):
outAssayNoteFile.write(str(assayKey) + TAB + \
note[i:i+ASSAY_NOTE_LENGTH] + TAB + \
loaddate + TAB + loaddate + CRT)
i = i + ASSAY_NOTE_LENGTH
# MGI Accession ID for the assay
outAccFile.write(str(accKey) + TAB + \
mgiPrefix + str(mgiKey) + TAB + \
mgiPrefix + TAB + \
str(mgiKey) + TAB + \
accLogicalDBKey + TAB + \
str(assayKey) + TAB + \
assayMgiTypeKey + TAB + \
accPrivate + TAB + \
accPreferred + TAB + \
str(createdByKey) + TAB + \
str(createdByKey) + TAB + \
loaddate + TAB + loaddate + CRT)
assayAssay[assayID] = assayKey
accKey = accKey + 1
mgiKey = mgiKey + 1
assayKey = assayKey + 1
# end of "for line in inAssayFile.readlines():"
return lineNum
def processFile():
global execSQL
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
probeID = tokens[0]
markerIDs = string.split(tokens[1], '|')
jnum = tokens[2]
relationship = tokens[3]
createdBy = tokens[4]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
probeKey = loadlib.verifyProbe(probeID, lineNum, errorFile)
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if probeKey == 0:
errorFile.write('Invalid Probe: %s\n' % (probeID))
error = 1
if referenceKey == 0:
errorFile.write('Invalid Reference: %s\n' % (jnum))
error = 1
if createdByKey == 0:
errorFile.write('Invalid Creator: %s\n\n' % (createdBy))
error = 1
# marker IDs
markerList = []
for markerID in markerIDs:
markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile)
if markerKey == 0:
errorFile.write('Invalid Marker: %s, %s\n' % (name, markerID))
error = 1
else:
markerList.append(markerKey)
# if errors, continue to next record
if error:
continue
# if no errors, process
for markerKey in markerList:
if markerList.count(markerKey) == 1:
markerFile.write('%s|%s|%d|%s|%s|%s|%s|%s\n' \
% (probeKey, markerKey, referenceKey, relationship, createdByKey, createdByKey, loaddate, loaddate))
execSQL.append(deleteSQL % (probeKey, markerKey))
else:
errorFile.write('Invalid Marker Duplicate: %s, %s\n' % (name, markerID))
def processFile():
global refKey, aliasKey
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
probeID = probeName = tokens[0]
jnum = tokens[1]
aliasList = string.split(tokens[2], '|')
createdBy = tokens[3]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
if probeID.find('MGI:') >= 0:
probeKey = loadlib.verifyProbe(probeID, lineNum, errorFile)
else:
probeKey, probeID = verifyProbe(probeName, lineNum, errorFile)
probeReferenceKey = verifyProbeReference(probeID, jnum, lineNum, errorFile)
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if probeKey == 0:
errorFile.write('Invalid Probe: %s\n' % (probeID))
error = 1
if referenceKey == 0:
errorFile.write('Invalid Reference: %s\n' % (jnum))
error = 1
#if probeReferenceKey == 0:
# errorFile.write('Invalid Probe Reference: %s, %s\n' % (probeID, jnum))
# error = 1
if createdByKey == 0:
errorFile.write('Invalid Creator: %s\n\n' % (createdBy))
error = 1
# if errors, continue to next record
if error:
continue
# if no errors, process
# create a new probe-reference key if one does not already exist
# else use the existing probe-reference key
if probeReferenceKey == 0:
refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' \
% (refKey, probeKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate))
aliasrefKey = refKey
refKey = refKey + 1
else:
#errorFile.write('Probe/Reference Already Exists: %s\n' % (tokens))
aliasrefKey = probeReferenceKey
# aliases
for alias in aliasList:
if len(alias) == 0:
continue
aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
% (aliasKey, aliasrefKey, alias, createdByKey, createdByKey, loaddate, loaddate))
aliasKey = aliasKey + 1
def processFile():
global alleleKey, refAssocKey, accKey, noteKey, mgiKey, annotKey
global alleleLookup
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = line[:-1].split('\t')
#print line
try:
markerID = tokens[0]
symbol = tokens[1]
name = tokens[2]
alleleStatus = tokens[3]
alleleType = tokens[4]
alleleSubtypes = tokens[5]
collectionKey = tokens[6]
germLine = tokens[7]
references = tokens[8]
strainOfOrigin = tokens[9]
mutantCellLine = tokens[10]
molecularNotes = tokens[11]
driverNotes = tokens[12]
ikmcNotes = tokens[13]
mutations = tokens[14]
inheritanceMode = tokens[15]
isMixed = tokens[16]
isExtinct = tokens[17]
createdBy = tokens[18]
createMCL = tokens[19]
createNote = tokens[20]
setStatus = tokens[21]
existingAlleleID = tokens[22]
ikmcSymbol = tokens[23]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
# creator
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if createdByKey == 0:
continue
# processing for IKMC-only
if len(createMCL) > 0 or len(createNote) > 0 or len(setStatus) > 0:
processFileIKMC(createMCL, createNote, setStatus, \
symbol, ikmcSymbol, mutantCellLine, ikmcNotes, \
createdByKey, existingAlleleID)
continue
# marker key
markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile)
# hard-coded
# _vocab_key = 73 (Marker-Allele Association Status)
# _term_key = 4268545 (Curated)
markerStatusKey = 4268545
# _vocab_key = 37 (Allele Status)
alleleStatusKey = loadlib.verifyTerm('', 37, alleleStatus, lineNum, errorFile)
# _vocab_key = 38 (Allele Type)
alleleTypeKey = loadlib.verifyTerm('', 38, alleleType, lineNum, errorFile)
# _vocab_key = 61 (Allele Transmission)
germLineKey = loadlib.verifyTerm('', 61, germLine, lineNum, errorFile)
# _vocab_key = 36 (Allele Molecular Mutation)
allMutations = mutations.split('|')
# _vocab_key = 35 (Allele Status)
inheritanceModeKey = loadlib.verifyTerm('', 35, inheritanceMode, lineNum, errorFile)
# strains
strainOfOriginKey = sourceloadlib.verifyStrain(strainOfOrigin, lineNum, errorFile)
# reference
refKey = loadlib.verifyReference(jnum, lineNum, errorFile)
# if errors, continue to next record
# errors are stored (via loadlib) in the .error log
if markerKey == 0 \
or markerStatusKey == 0 \
or alleleStatusKey == 0 \
or alleleTypeKey == 0 \
or germLineKey == 0 \
or allMutations == 0 \
or inheritanceModeKey == 0 \
or strainOfOriginKey == 0 \
or refKey == 0 \
or createdByKey == 0:
continue
# if no errors, process the allele
# not specified/testing
#collectionKey = 11025586
# allele (master)
alleleFile.write('%d|%s|%s|%s|%s|%s|%s|%s|%s|%s|0|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (alleleKey, markerKey, strainOfOriginKey, inheritanceModeKey, alleleTypeKey, \
alleleStatusKey, germLineKey, collectionKey, symbol, name, \
isExtinct, isMixed, refKey, markerStatusKey, \
createdByKey, createdByKey, createdByKey, loaddate, loaddate, loaddate))
# molecular mutation
for mutation in allMutations:
mutationKey = loadlib.verifyTerm('', 36, mutation, lineNum, errorFile)
mutationFile.write('%s|%s|%s|%s\n' \
% (alleleKey, mutationKey, loaddate, loaddate))
#
# allele references
#
allReferences = references.split('||')
for reference in allReferences:
refType, refID = reference.split('|')
refKey = loadlib.verifyReference(refID, lineNum, errorFile)
if refType == 'Original':
refAssocTypeKey = 1011
elif refType == 'Transmission':
refAssocTypeKey = 1023
elif refType == 'Molecular':
refAssocTypeKey = 1012
refFile.write('%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (refAssocKey, refKey, alleleKey, mgiTypeKey, refAssocTypeKey, \
createdByKey, createdByKey, loaddate, loaddate))
refAssocKey = refAssocKey + 1
#
# allele subtypes
#
allSubtypes = alleleSubtypes.split('|')
for s in allSubtypes:
# _vocab_key = 93 (Allele Subtype)
alleleSubtypeKey = loadlib.verifyTerm('', 93, s, lineNum, errorFile)
annotFile.write('%s|%s|%s|%s|%s|%s|%s\n' \
% (annotKey, annotTypeKey, alleleKey, alleleSubtypeKey, \
qualifierKey, loaddate, loaddate))
annotKey = annotKey + 1
#
# mutant cell line
#
if len(mutantCellLine) > 0:
addMutantCellLine(alleleKey, mutantCellLine, createdByKey)
# MGI Accession ID for the allelearker
accFile.write('%s|%s%d|%s|%s|1|%d|%d|0|1|%s|%s|%s|%s\n' \
% (accKey, mgiPrefix, mgiKey, mgiPrefix, mgiKey, alleleKey, mgiTypeKey, \
createdByKey, createdByKey, loaddate, loaddate))
# storing data in MGI_Note/MGI_NoteChunk
# molecular notes
mgiNoteSeqNum = 1
if len(molecularNotes) > 0:
noteFile.write('%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (noteKey, alleleKey, mgiNoteObjectKey, mgiMolecularNoteTypeKey, \
createdByKey, createdByKey, loaddate, loaddate))
noteChunkFile.write('%s|%s|%s|%s|%s|%s|%s\n' \
% (noteKey, mgiNoteSeqNum, molecularNotes, createdByKey, createdByKey, loaddate, loaddate))
noteKey = noteKey + 1
# driver notes
# TR12662/MGI_Relationship._Category_key = 1006
# removed noteFile code
# place hodler for MGI_Relationship code
# the IKMC is the only product using this and IKMC does not add any driver note
#mgiNoteSeqNum = 1
#if len(driverNotes) > 0:
# ikmc notes
useIKMCnotekey = 0
if len(ikmcNotes) > 0:
noteFile.write('%s|%s|%s|%s|%s|%s|%s|%s\n' \
% (noteKey, alleleKey, mgiNoteObjectKey, mgiIKMCNoteTypeKey, \
createdByKey, createdByKey, loaddate, loaddate))
noteChunkFile.write('%s|%s|%s|%s|%s|%s|%s\n' \
% (noteKey, 1, ikmcNotes, createdByKey, createdByKey, loaddate, loaddate))
useIKMCnotekey = noteKey
noteKey = noteKey + 1
# Print out a new text file and attach the new MGI Allele IDs as the last field
if createdBy == 'ikmc_alleleload':
newAlleleFile.write('%s\t%s%s\t%s\n' \
% (mgi_utils.prvalue(ikmcNotes), \
mgi_utils.prvalue(mgiPrefix), mgi_utils.prvalue(mgiKey), \
mgi_utils.prvalue(ikmcSymbol)))
else:
newAlleleFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%s\n' \
% (mgi_utils.prvalue(markerID), \
mgi_utils.prvalue(symbol), \
mgi_utils.prvalue(name), \
mgi_utils.prvalue(alleleStatus), \
mgi_utils.prvalue(alleleType), \
mgi_utils.prvalue(alleleSubtype), \
mgi_utils.prvalue(collection), \
mgi_utils.prvalue(germLine), \
mgi_utils.prvalue(references), \
mgi_utils.prvalue(strainOfOrigin), \
mgi_utils.prvalue(mutantCellLine), \
mgi_utils.prvalue(allMutations), \
mgi_utils.prvalue(inheritanceMode), \
mgi_utils.prvalue(isMixed), \
mgi_utils.prvalue(isExtinct), \
mgi_utils.prvalue(refKey), \
mgi_utils.prvalue(markerStatusKey), \
mgi_utils.prvalue(createdBy), \
mgi_utils.prvalue(mgiPrefix), mgi_utils.prvalue(mgiKey)))
# save symbol/alleleKey/ikmc note key
alleleLookup[symbol] = []
alleleLookup[symbol].append((alleleKey, useIKMCnotekey, mgiPrefix + str(mgiKey)))
accKey = accKey + 1
mgiKey = mgiKey + 1
alleleKey = alleleKey + 1
# end of "for line in inputFile.readlines():"
#
# Update the AccessionMax value
#
if not DEBUG:
db.sql('select * from ACC_setMax(%d)' % (lineNum), None)
db.commit()
def processFile():
global primerKey, refKey, aliasKey, accKey, mgiKey
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
markerSymbol = tokens[0] # not used
markerIDs = string.split(tokens[1], '|')
name = tokens[2]
jnum = tokens[3]
regionCovered = tokens[4]
sequence1 = tokens[5]
sequence2 = tokens[6]
productSize = tokens[7]
notes = tokens[8]
sequenceIDs = tokens[9]
aliasList = string.split(tokens[10], '|')
createdBy = tokens[11]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
# marker IDs
markerList = []
for markerID in markerIDs:
markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile)
if len(markerID) > 0 and markerKey == 0:
errorFile.write('Invalid Marker: %s, %s\n' % (name, markerID))
error = 1
elif len(markerID) > 0:
markerList.append(markerKey)
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
# sequence IDs
seqAccList = string.split(sequenceIDs, '|')
# if errors, continue to next record
if error:
continue
# if no errors, process the primer
primerFile.write('%d\t%s\t\t%d\t%d\t%s\t%s\t%s\t%s\t\t\t%s\t%s\t%s\t%s\t%s\n' \
% (primerKey, name, NA, vectorKey, segmentTypeKey, mgi_utils.prvalue(sequence1), \
mgi_utils.prvalue(sequence2), mgi_utils.prvalue(regionCovered), mgi_utils.prvalue(productSize), \
createdByKey, createdByKey, loaddate, loaddate))
for markerKey in markerList:
if markerList.count(markerKey) == 1:
markerFile.write('%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\n' \
% (primerKey, markerKey, referenceKey, relationship, createdByKey, createdByKey, loaddate, loaddate))
else:
errorFile.write('Invalid Marker Duplicate: %s, %s\n' %
(name, markerID))
# loaddate))
refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' %
(refKey, primerKey, referenceKey, createdByKey,
createdByKey, loaddate, loaddate))
# aliases
for alias in aliasList:
if len(alias) == 0:
continue
aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
% (aliasKey, refKey, alias, createdByKey, createdByKey, loaddate, loaddate))
aliasKey = aliasKey + 1
# MGI Accession ID for the marker
accFile.write('%s\t%s%d\t%s\t%s\t1\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \
% (accKey, mgiPrefix, mgiKey, mgiPrefix, mgiKey, primerKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate))
newPrimerFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%d\n' \
% (markerSymbol, string.join(markerIDs, '|'), name, jnum, regionCovered, sequence1, sequence2, productSize, notes, sequenceIDs, createdBy, mgiPrefix, mgiKey))
accKey = accKey + 1
mgiKey = mgiKey + 1
# sequence accession ids
for acc in seqAccList:
if len(acc) == 0:
continue
prefixPart, numericPart = accessionlib.split_accnum(acc)
accFile.write('%s\t%s\t%s\t%s\t%s\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \
% (accKey, acc, prefixPart, numericPart, logicalDBKey, primerKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate))
accRefFile.write('%s\t%s\t%s\t%s\t%s\t%s\n' \
% (accKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate))
accKey = accKey + 1
# notes
if len(notes) > 0:
noteFile.write('%s|1\t%s\t%s\t%s\n' \
% (primerKey, notes, loaddate, loaddate))
refKey = refKey + 1
primerKey = primerKey + 1
# end of "for line in inputFile.readlines():"
#
# Update the AccessionMax value
#
if not DEBUG:
db.sql('select * from ACC_setMax (%d)' % (lineNum), None)
def init():
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes local DBMS parameters
# 3. Initializes global file descriptors/file names
# 4. Initializes global keys
#
# returns:
#
global diagFileName, errorFileName, synFileName
global inputFile, diagFile, errorFile, synFile
global mgiTypeKey, createdByKey, referenceKey
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
head, tail = os.path.split(inputFileName)
diagFileName = logDir + '/' + tail + '.diagnostics'
errorFileName = logDir + '/' + tail + '.error'
synFileName = 'MGI_Synonym.bcp'
print inputFileName
print logDir
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
synFile = open(outputDir + '/' + synFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % synFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
diagFile.write('Object Type: %s\n' % (mgiType))
diagFile.write('Input File: %s\n' % (inputFileName))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
mgiTypeKey = loadlib.verifyMGIType(mgiType, 0, errorFile)
createdByKey = loadlib.verifyUser(createdBy, 0, errorFile)
# if reference is J:0, then no reference is given
if jnum == 'J:0':
referenceKey = ''
else:
referenceKey = loadlib.verifyReference(jnum, 0, errorFile)
# exit if we can't resolve mgiType, createdBy or jnum
if mgiTypeKey == 0 or \
createdByKey == 0 or \
referenceKey == 0:
exit(1)
if mode == 'reload':
print 'mode is: %s, deleting synonyms' % mode
sys.stdout.flush()
db.sql('delete from MGI_Synonym ' + \
'where _MGIType_key = %d ' % (mgiTypeKey) + \
'and _CreatedBy_key = %d ' % (createdByKey), None)
def processFile():
global probeKey, refKey, aliasKey, accKey, mgiKey
lineNum = 0
# For each line in the input file
for line in inputFile.readlines():
error = 0
lineNum = lineNum + 1
# Split the line into tokens
tokens = string.split(line[:-1], '\t')
try:
name = tokens[0]
jnum = tokens[1]
parentID = tokens[2]
sourceName = tokens[3]
organism = tokens[4]
strain = tokens[5]
tissue = tokens[6]
gender = tokens[7]
cellLine = tokens[8]
age = tokens[9]
vectorType = tokens[10]
segmentType = tokens[11]
regionCovered = tokens[12]
insertSite = tokens[13]
insertSize = tokens[14]
markerIDs = string.split(tokens[15], '|')
relationship = tokens[16]
sequenceIDs = tokens[17]
aliasList = string.split(tokens[18], '|')
notes = tokens[19]
rawnotes = tokens[20]
createdBy = tokens[21]
except:
exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line))
isParent = 0
isSource = 0
parentProbeKey = ''
sourceKey = 0
if parentID != '':
isParent = 1
if sourceName != '':
isSource = 1
if not isParent and not isSource:
organismKey = sourceloadlib.verifyOrganism(organism, lineNum,
errorFile)
strainKey = sourceloadlib.verifyStrain(strain, lineNum, errorFile)
tissueKey = sourceloadlib.verifyTissue(tissue, lineNum, errorFile)
genderKey = sourceloadlib.verifyGender(gender, lineNum, errorFile)
cellLineKey = sourceloadlib.verifyCellLine(cellLine, lineNum,
errorFile)
vectorKey = sourceloadlib.verifyVectorType(vectorType, lineNum,
errorFile)
segmentTypeKey = sourceloadlib.verifySegmentType(
segmentType, lineNum, errorFile)
sourceKey = sourceloadlib.verifySource(segmentTypeKey, \
vectorKey, organismKey, strainKey, \
tissueKey, genderKey, cellLineKey, age, lineNum, errorFile)
if organismKey == 0 or strainKey == 0 or tissueKey == 0 or \
genderKey == 0 or cellLineKey == 0 or vectorKey == 0 or \
segmentTypeKey == 0 or sourceKey == 0:
errorFile.write('%s, %s, %s, %s, %s, %s, %s, %s\n' %
(segmentType, vectorType, organism, strain,
tissue, gender, cellLine, age))
error = 1
elif not isParent and isSource:
vectorKey = sourceloadlib.verifyVectorType(vectorType, lineNum,
errorFile)
segmentTypeKey = sourceloadlib.verifySegmentType(
segmentType, lineNum, errorFile)
sourceKey = sourceloadlib.verifyLibrary(sourceName, lineNum,
errorFile)
if vectorKey == 0 or segmentTypeKey == 0 or sourceKey == 0:
error = 1
# parent from = yes, source given = yes or no (ignored)
else:
parentProbeKey, sourceKey = verifyParentProbe(
parentID, lineNum, errorFile)
vectorKey = sourceloadlib.verifyVectorType(vectorType, lineNum,
errorFile)
segmentTypeKey = sourceloadlib.verifySegmentType(
segmentType, lineNum, errorFile)
if parentProbeKey == 0 or sourceKey == 0 or vectorKey == 0 or segmentTypeKey == 0:
error = 1
referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile)
createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile)
if referenceKey == 0:
errorFile.write('Invalid Reference: %s\n' % (jnum))
error = 1
if createdByKey == 0:
errorFile.write('Invalid Creator: %s\n\n' % (createdBy))
error = 1
# marker IDs
markerList = []
for markerID in markerIDs:
markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile)
if len(markerID) > 0 and markerKey == 0:
errorFile.write('Invalid Marker: %s, %s\n' % (name, markerID))
error = 1
elif len(markerID) > 0:
markerList.append(markerKey)
# sequence IDs
seqAccDict = {}
for seqID in string.split(sequenceIDs, '|'):
if len(seqID) > 0:
[logicalDB, acc] = string.split(seqID, ':')
logicalDBKey = loadlib.verifyLogicalDB(logicalDB, lineNum,
errorFile)
if logicalDBKey > 0:
seqAccDict[acc] = logicalDBKey
# if errors, continue to next record
if error:
continue
# if no errors, process the probe
probeFile.write('%d\t%s\t%s\t%s\t%s\t%s\t\t\t%s\t%s\t%s\t\t%s\t%s\t%s\t%s\n' \
% (probeKey, name, parentProbeKey, sourceKey, vectorKey, segmentTypeKey, mgi_utils.prvalue(regionCovered), \
mgi_utils.prvalue(insertSite), mgi_utils.prvalue(insertSize), createdByKey, createdByKey, loaddate, loaddate))
for markerKey in markerList:
if markerList.count(markerKey) == 1:
markerFile.write('%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\n' \
% (probeKey, markerKey, referenceKey, relationship, createdByKey, createdByKey, loaddate, loaddate))
else:
errorFile.write('Invalid Marker Duplicate: %s, %s\n' %
(name, markerID))
refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' \
% (refKey, probeKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate))
# aliases
for alias in aliasList:
if len(alias) == 0:
continue
aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
% (aliasKey, refKey, alias, createdByKey, createdByKey, loaddate, loaddate))
aliasKey = aliasKey + 1
# MGI Accession ID for the marker
accFile.write('%s\t%s%d\t%s\t%s\t1\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \
% (accKey, mgiPrefix, mgiKey, mgiPrefix, mgiKey, probeKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate))
# Print out a new text file and attach the new MGI Probe IDs as the last field
newProbeFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%d\n' \
% (name, jnum, \
mgi_utils.prvalue(sourceName), \
organism, \
mgi_utils.prvalue(strain), \
mgi_utils.prvalue(tissue), \
mgi_utils.prvalue(gender), \
mgi_utils.prvalue(cellLine), \
mgi_utils.prvalue(age), \
mgi_utils.prvalue(vectorType), \
mgi_utils.prvalue(segmentType), \
mgi_utils.prvalue(regionCovered) + \
mgi_utils.prvalue(insertSite), \
mgi_utils.prvalue(insertSize), \
string.join(markerIDs, '|'), \
relationship, \
mgi_utils.prvalue(sequenceIDs), \
string.join(aliasList, '|'), \
mgi_utils.prvalue(notes), \
createdBy, mgiPrefix, mgiKey))
# Print out a raw note file
if len(rawnotes) > 0:
rawNoteFile.write('%s%d\t%s\n' % (mgiPrefix, mgiKey, rawnotes))
# Notes
if len(notes) > 0:
noteFile.write('%s\t%s\t%s\t%s\n' %
(probeKey, notes, loaddate, loaddate))
accKey = accKey + 1
mgiKey = mgiKey + 1
# sequence accession ids
for acc in seqAccDict.keys():
prefixPart, numericPart = accessionlib.split_accnum(acc)
accFile.write('%s\t%s\t%s\t%s\t%s\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \
% (accKey, acc, prefixPart, numericPart, seqAccDict[acc], probeKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate))
accRefFile.write('%s\t%s\t%s\t%s\t%s\t%s\n' \
% (accKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate))
accKey = accKey + 1
refKey = refKey + 1
probeKey = probeKey + 1
# end of "for line in inputFile.readlines():"
#
# Update the AccessionMax value
#
if not DEBUG:
db.sql('select * from ACC_setMax (%d)' % (lineNum), None)