def writeAccBCP(): ''' # requires: # # effects: # Creates approrpriate BCP records # # returns: # nothing # ''' global accKey, userKey # records that require a reference results = db.sql('select _Object_key, _LogicalDB_key, accID, private, geneID ' + \ 'from WRK_EntrezGene_Bucket0 ' + \ 'where taxID = %s and refRequired = 1 ' % (taxId), 'auto') for r in results: if r['_Object_key'] == -1: objectKey = geneIDtoMarkerKey[r['geneID']] else: objectKey = r['_Object_key'] prefixPart, numericPart = accessionlib.split_accnum(r['accID']) accFile.write('%d|%s|%s|%s|%d|%d|%s|%d|1|%s|%s|%s|%s\n' % (accKey, r['accID'], mgi_utils.prvalue(prefixPart), mgi_utils.prvalue(numericPart), r['_LogicalDB_key'], objectKey, mgiTypeKey, r['private'], userKey, userKey, loaddate, loaddate)) accrefFile.write('%d|%s|%s|%s|%s|%s\n' % (accKey, referenceKey, userKey, userKey, loaddate, loaddate)) accKey = accKey + 1 # records that don't require a reference results = db.sql('select _Object_key, _LogicalDB_key, accID, private, geneID ' + \ 'from WRK_EntrezGene_Bucket0 ' + \ 'where taxID = %s and refRequired = 0' % (taxId), 'auto') for r in results: if r['_Object_key'] == -1: objectKey = geneIDtoMarkerKey[r['geneID']] else: objectKey = r['_Object_key'] prefixPart, numericPart = accessionlib.split_accnum(r['accID']) accFile.write('%d|%s|%s|%s|%d|%d|%s|%d|1|%s|%s|%s|%s\n' % (accKey, r['accID'], mgi_utils.prvalue(prefixPart), mgi_utils.prvalue(numericPart), r['_LogicalDB_key'], objectKey, mgiTypeKey, r['private'], userKey, userKey, loaddate, loaddate)) accKey = accKey + 1
def createBCPFile(): global accKey print 'Create the bcp file for the GENSAT associations' # # Find the marker key that the EntrezGene ID should be associated with. # Do not make an association for any EntrezGene IDs that are on the # discrepancy report. # cmds = [] cmds.append('select t.entrezgeneID, a._Object_key as markerKey ' + \ 'from ' + tempTable + ' t, ACC_Accession a ' + \ 'where lower(t.entrezgeneID) = lower(a.accID) and ' + \ 'a._MGIType_key = ' + str(markerMGITypeKey) + ' and ' + \ 'a._LogicalDB_key = ' + str(egLogicalDBKey) + ' ' + \ 'order by t.entrezgeneID') results = db.sql(cmds, 'auto') count = 0 # # Write the records to the bcp file. # for r in results[0]: entrezgeneID = r['entrezgeneID'] markerKey = r['markerKey'] # # Skip the EntrezGene ID if it was written to the discrepancy report. # if badIDs.has_key(entrezgeneID): continue # # Get the prefix and numeric parts of the EntrezGene ID and write # a record to the bcp file. # (prefixPart, numericPart) = accessionlib.split_accnum(entrezgeneID) fpAccBCPFile.write(str(accKey) + TAB + \ entrezgeneID + TAB + \ prefixPart + TAB + \ str(numericPart) + TAB + \ str(gensatLogicalDBKey) + TAB + \ str(markerKey) + TAB + \ str(markerMGITypeKey) + TAB + \ PRIVATE + TAB + PREFERRED + TAB + \ str(createdByKey) + TAB + \ str(createdByKey) + TAB + \ loadDate + TAB + \ loadDate + NL) count = count + 1 accKey = accKey + 1 print 'Number of GENSAT associations: ' + str(count) return
def createBCPFile (): global accKey print 'Create the bcp file for the GENSAT associations' # # Find the marker key that the EntrezGene ID should be associated with. # Do not make an association for any EntrezGene IDs that are on the # discrepancy report. # cmds = [] cmds.append('select t.entrezgeneID, a._Object_key as markerKey ' + \ 'from ' + tempTable + ' t, ACC_Accession a ' + \ 'where lower(t.entrezgeneID) = lower(a.accID) and ' + \ 'a._MGIType_key = ' + str(markerMGITypeKey) + ' and ' + \ 'a._LogicalDB_key = ' + str(egLogicalDBKey) + ' ' + \ 'order by t.entrezgeneID') results = db.sql(cmds,'auto') count = 0 # # Write the records to the bcp file. # for r in results[0]: entrezgeneID = r['entrezgeneID'] markerKey = r['markerKey'] # # Skip the EntrezGene ID if it was written to the discrepancy report. # if badIDs.has_key(entrezgeneID): continue # # Get the prefix and numeric parts of the EntrezGene ID and write # a record to the bcp file. # (prefixPart,numericPart) = accessionlib.split_accnum(entrezgeneID) fpAccBCPFile.write(str(accKey) + TAB + \ entrezgeneID + TAB + \ prefixPart + TAB + \ str(numericPart) + TAB + \ str(gensatLogicalDBKey) + TAB + \ str(markerKey) + TAB + \ str(markerMGITypeKey) + TAB + \ PRIVATE + TAB + PREFERRED + TAB + \ str(createdByKey) + TAB + \ str(createdByKey) + TAB + \ loadDate + TAB + \ loadDate + NL) count = count + 1 accKey = accKey + 1 print 'Number of GENSAT associations: ' + str(count) return
def writeMGPOutput(): # Purpose: writes to Accession, AccessionReference & StrainMarker # BCP file and Gene Model and GM Assoc files if there are no errors # Returns: 1 if error, else 0 # Assumes: file descriptors have been initialized # Effects: writes to the file system # Throws: Nothing global nextSMKey, nextAccKey, mgpSkipCt, totalLoadedCt # for markers with >1 strain specific MGP ID # report and load a strain marker and a accession for each MGP ID strainMarkerInputList = qcDict['mgi_mgp'] for strainMarkerInputDict in strainMarkerInputList: for strain in strainMarkerInputDict: strainMarkerObjectsList = strainMarkerInputDict[strain] for coordsForMarkerList in strainMarkerObjectsList: for strainMarkerObject in coordsForMarkerList: # write out to bcp file mgpID = strainMarkerObject.mgpID mgiID = strainMarkerObject.markerID markerKey = strainMarkerObject.markerKey if mgiID.find( 'TEMP:' ) == 0: # temp ID for no marker strain marker markerKey = '' strainKey = strainMarkerObject.strainKey chr = strainMarkerObject.chr start = strainMarkerObject.start end = strainMarkerObject.end strand = strainMarkerObject.strand description = strainMarkerObject.description biotype = strainMarkerObject.biotype totalLoadedCt += 1 fpStrainMarkerFile.write( '%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' % (nextSMKey, TAB, strainKey, TAB, markerKey, TAB, mgpRefsKey, TAB, userKey, TAB, userKey, TAB, loaddate, TAB, loaddate, CRT)) prefixPart, numericPart = accessionlib.split_accnum(mgpID) fpAccFile.write('%s%s%s%s%s%s%s%s%s%s%s%s%s%s0%s1%s%s%s%s%s%s%s%s%s' \ % (nextAccKey, TAB, mgpID, TAB, prefixPart, TAB, numericPart, TAB, mgpLDBKey, TAB, nextSMKey, TAB, mgiTypeKey, TAB, TAB, TAB, userKey, TAB, userKey, TAB, loaddate, TAB, loaddate, CRT)) fpAccRefFile.write('%s%s%s%s%s%s%s%s%s%s%s%s' \ % (nextAccKey, TAB, mgpRefsKey, TAB, userKey, TAB, userKey, TAB, loaddate, TAB, loaddate, CRT)) fpGmMgpFile.write('%s%s%s%s%s%s%s%s%s%s%s%s' % (mgpID, TAB, chr, TAB, start, TAB, end, TAB, strand, TAB, description, CRT)) fpBiotypeMgpFile.write('%s%s%s%s' % (mgpID, TAB, biotype, CRT)) nextSMKey += 1 nextAccKey += 1 return 0
def processSusceptibility(): # do formatted file doFileName = None # do file pointer doFile = None # insert statement INSERT_ACCESSION = '''insert into ACC_Accession values ((select max(_Accession_key) + 1 from ACC_Accession), '%s', '%s', %s, 15, %s, 13, 0, 0) ''' doFileName = os.environ['OBO_FILE'] doFile = open(doFileName, 'r') omimIdValue = 'id: OMIM:' relValue = 'relationship: RO:0003304' skipValue = 'OMIM:000000' foundOMIM = 0 for line in doFile.readlines(): # find [Term] # find relationship: RO:0003304 if line == '[Term]': foundOMIM = 0 elif line[:9] == omimIdValue: omimId = line[4:-1] if omimId == skipValue: continue foundOMIM = 1 elif foundOMIM and line[:24] == relValue: tokens = line[25:-1].split(' ') doId = tokens[0] prefixPart, numericPart = accessionlib.split_accnum(omimId) objectKey = loadlib.verifyObject(doId, 13, None, None, None) addSQL = INSERT_ACCESSION % (omimId, prefixPart, numericPart, objectKey) db.sql(addSQL, None) else: continue doFile.close() db.commit() return 0
def process(): global propertiesDict, expCount, loadedCount, inDbCount, invalidSampleCountDict global invalidReleaseDateDict, invalidUpdateDateDict, noIdList global nextExptKey, nextAccKey, nextExptVarKey, nextPropKey global updateExptCount for f in jFile['experiments']['experiment']: expCount += 1 # definitions with SUPERSERIES text get different evaluation state # than the load default and evalution date and evaluated by are set # by the load (default null) isSuperSeries = 0 evalStateToUseKey = defaultEvalStateTermKey try: # description is str.or list allDescription = f['description'] description = allDescription['text'] # experiment, onea # US108 'clean up URLs that appear in description field'. All # URLs that need to be cleaned up are the listType description # example of element in a description list with URL that we # need to parse: # {'a': {'href': 'http://lgsun.grc.nia.nih.gov/ANOVA/', 'target': '_blank', '$': 'http://lgsun.grc.nia.nih.gov/ANOVA/'}} if type(description) == list: listDescript = '' for d in description: if type(d) == dict: if 'a' in d: url = d['a']['$'] listDescript = listDescript + url # skip these: {"br":null} elif 'br' in d: continue else: listDescript = listDescript + str(d) description = listDescript except: description = '' if description == None: # {'text': None, 'id': None} description = '' description = str.strip(description) if description.find(SUPERSERIES) != -1: evalStateToUseKey = altEvalStateTermKey isSuperSeries = 1 try: name = f['name'] if type(name) == list: name = '|'.join(name) except: name = '' name = str.strip(name) try: primaryID = str.strip(f['accession']) # accession except: primaryID = '' try: sampleCount = f['samples'] # property, one except: sampleCount = '' try: releasedate = f['releasedate'] # experiment, one except: releasedate = '' try: # experimentalfactor.name # list or dict expFactor = f['experimentalfactor'] if type(expFactor) == dict: expFactorList = [expFactor] else: expFactorList = expFactor expFactorSet = set() for e in expFactorList: #property, many stored individ. # weed out dups expFactorSet.add(e['name']) expFactorList = list(expFactorSet) except: expFactorList = [] try: lastupdatedate = f['lastupdatedate'] # experiment, one except: lastupdatedate = '' try: # provider.contact, dictionary or list of dictionaries; need # to remove exact dups providerList = [] if type(f['provider']) != list: providerList = [f['provider']['contact']] else: for p in f['provider']: if p['contact'] != None: providerList.append(p['contact']) providerSet = set(providerList) providerList = list(providerSet) except: providerList = [] try: # experimenttype is str.or list, property, # many stored individ if type(f['experimenttype']) != list: experimenttypeList = [f['experimenttype']] else: experimenttypeList = f['experimenttype'] except: experimenttypeList = [] # pick first valid experiment type and translate it to populate the # exptype key exptTypeKey = 0 for exp in experimenttypeList: if exp in exptTypeTransDict: exptTypeKey = exptTypeTransDict[exp] break if exptTypeKey == 0: exptTypeKey = exptTypeNRKey # Not Resolved try: # PubMed IDs - bibliography.accession # TR13116/check for duplicate pubmedids bibliographyList = [] if type(f['bibliography']) == dict: # dictionary if str(f['bibliography']['accession']) not in bibliographyList: bibliographyList.append(str( f['bibliography']['accession'])) else: # ListType for b in f['bibliography']: # for each dict in the list if 'accession' in b: if str(b['accession']) not in bibliographyList: bibliographyList.append(str(b['accession'])) except: bibliographyList = [] # the template for properties: propertyTemplate = "#====#%s%s%s#=#%s%s%s%s%s#==#%s#===#%s%s%s%s%s%s%s%s%s" % ( TAB, propTypeKey, TAB, TAB, nextExptKey, TAB, mgiTypeKey, TAB, TAB, TAB, userKey, TAB, userKey, TAB, loadDate, TAB, loadDate, CRT) propertyUpdateTemplate = "#====#%s%s%s#=#%s#=====#%s%s%s#==#%s#===#%s%s%s%s%s%s%s%s%s" % ( TAB, propTypeKey, TAB, TAB, TAB, mgiTypeKey, TAB, TAB, TAB, userKey, TAB, userKey, TAB, loadDate, TAB, loadDate, CRT) # # update pubmed ID properties, if this ID already in the database # if primaryID in primaryIdDict: inDbCount += 1 # not all experiments have pubmed IDs if primaryID in pubMedByExptDict: # get the list of pubmed Ids for this expt in the database dbBibList = pubMedByExptDict[primaryID] # get the set of incoming pubmed IDs not in the database newSet = set(bibliographyList).difference(set(dbBibList)) # if we have new pubmed IDs, add them to the database if newSet: updateExpKey = primaryIdDict[primaryID] # get next sequenceNum for this expt's pubmed ID # in the database results = db.sql( '''select max(sequenceNum) + 1 as nextNum from MGI_Property p where p._Object_key = %s and p._PropertyTerm_key = 20475430 and p._PropertyType_key = 1002''' % updateExpKey, 'auto') nextSeqNum = results[0]['nextNum'] if newSet: updateExptCount += 1 for b in newSet: toLoad = propertyUpdateTemplate.replace( '#=#', str(pubmedPropKey)).replace( '#==#', str(b)).replace('#===#', str(nextSeqNum)).replace( '#====#', str(nextPropKey)).replace( '#=====#', str(updateExpKey)) fpPropertyBcp.write(toLoad) nextPropKey += 1 # continue so we don't dup what is in the db continue prefixPartPrimary, numericPartPrimary = accessionlib.split_accnum( primaryID) # # Do QC checks # If there are errors, skip to the next experiment if doQcChecks(primaryID, name, sampleCount, releasedate, lastupdatedate): continue # calculate secondary GEO ID for AE GEO IDs geoID = calculateGeoId(primaryID) # # now write out to bcp files # loadedCount += 1 # GXD_Experiment # many optional nulls - create the insert string line = '%s%s%s%s' % (nextExptKey, TAB, sourceKey, TAB) if name != '': line = line + name + TAB if description != '' and description != None: line = line + description + TAB else: line = line + TAB if releasedate != '': line = line + releasedate + TAB else: line = line + TAB if lastupdatedate != '': line = line + lastupdatedate + TAB else: line = line + TAB # evaluated data is today if isSuperSeries: line = line + loadDate + TAB else: # evaluated_date is null line = line + TAB line = line + str(evalStateToUseKey) + TAB if isSuperSeries: line = line + str(altCurStateTermKey) + TAB else: line = line + str(curStateTermKey) + TAB line = line + str(studyTypeTermKey) + TAB line = line + str(exptTypeKey) + TAB # evalByKey is null unless isSuperSeries is true then # it is load user if isSuperSeries: line = line + str(userKey) + TAB else: line = line + TAB # initialCurByKey, lastCurByKey, initialCurDate, lastCurDate # all null line = line + TAB + TAB + TAB + TAB # created and modified by line = line + str(userKey) + TAB + str(userKey) + TAB # creation and modification date line = line + loadDate + TAB + loadDate + CRT fpExperimentBcp.write(line) # Primary Accession fpAccBcp.write( '%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' % (nextAccKey, TAB, primaryID, TAB, prefixPartPrimary, TAB, numericPartPrimary, TAB, aeLdbKey, TAB, nextExptKey, TAB, mgiTypeKey, TAB, private, TAB, isPreferred, TAB, userKey, TAB, userKey, TAB, loadDate, TAB, loadDate, CRT)) nextAccKey += 1 # Secondary Accession if geoID != '': prefixPartSecondary, numericPartSecondary = accessionlib.split_accnum( geoID) fpAccBcp.write( '%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' % (nextAccKey, TAB, geoID, TAB, prefixPartSecondary, TAB, numericPartSecondary, TAB, geoLdbKey, TAB, nextExptKey, TAB, mgiTypeKey, TAB, private, TAB, notPreferred, TAB, userKey, TAB, userKey, TAB, loadDate, TAB, loadDate, CRT)) nextAccKey += 1 # Variable fpVariableBcp.write( '%s%s%s%s%s%s' % (nextExptVarKey, TAB, nextExptKey, TAB, exptVariableTermKey, CRT)) nextExptVarKey += 1 # # Properties # # name (0,1, pipe-delim) # sampleCount (0,1) # expFactorList (0-n) # providerList (0-n) # experimenttypeList (0-n) # bibiliographyList (0-n) # # propName, value and sequenceNum to be filled in later if name != '': toLoad = propertyTemplate.replace('#=#', str(namePropKey)).replace( '#==#', name).replace('#===#', '1').replace('#====#', str(nextPropKey)) fpPropertyBcp.write(toLoad) nextPropKey += 1 if sampleCount != '': toLoad = propertyTemplate.replace( '#=#', str(sampleCountPropKey)).replace( '#==#', str(sampleCount)).replace('#===#', '1').replace( '#====#', str(nextPropKey)) fpPropertyBcp.write(toLoad) nextPropKey += 1 seqNumCt = 1 for e in expFactorList: toLoad = propertyTemplate.replace( '#=#', str(expFactorPropKey)).replace('#==#', e).replace( '#===#', str(seqNumCt)).replace('#====#', str(nextPropKey)) fpPropertyBcp.write(toLoad) seqNumCt += 1 nextPropKey += 1 seqNumCt = 1 for p in providerList: toLoad = propertyTemplate.replace( '#=#', str(contactNamePropKey)).replace('#==#', p).replace( '#===#', str(seqNumCt)).replace('#====#', str(nextPropKey)) fpPropertyBcp.write(toLoad) seqNumCt += 1 nextPropKey += 1 seqNumCt = 1 for e in experimenttypeList: toLoad = propertyTemplate.replace( '#=#', str(expTypePropKey)).replace('#==#', e).replace( '#===#', str(seqNumCt)).replace('#====#', str(nextPropKey)) fpPropertyBcp.write(toLoad) seqNumCt += 1 nextPropKey += 1 seqNumCt = 1 for b in bibliographyList: toLoad = propertyTemplate.replace( '#=#', str(pubmedPropKey)).replace('#==#', str(b)).replace( '#===#', str(seqNumCt)).replace('#====#', str(nextPropKey)) fpPropertyBcp.write(toLoad) seqNumCt += 1 nextPropKey += 1 nextExptKey += 1 return
def processFile(): global primerKey, refKey, aliasKey, accKey, mgiKey lineNum = 0 # For each line in the input file for line in inputFile.readlines(): error = 0 lineNum = lineNum + 1 # Split the line into tokens tokens = string.split(line[:-1], '\t') try: markerSymbol = tokens[0] # not used markerIDs = string.split(tokens[1], '|') name = tokens[2] jnum = tokens[3] regionCovered = tokens[4] sequence1 = tokens[5] sequence2 = tokens[6] productSize = tokens[7] notes = tokens[8] sequenceIDs = tokens[9] aliasList = string.split(tokens[10], '|') createdBy = tokens[11] except: exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line)) # marker IDs markerList = [] for markerID in markerIDs: markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile) if len(markerID) > 0 and markerKey == 0: errorFile.write('Invalid Marker: %s, %s\n' % (name, markerID)) error = 1 elif len(markerID) > 0: markerList.append(markerKey) referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile) createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile) # sequence IDs seqAccList = string.split(sequenceIDs, '|') # if errors, continue to next record if error: continue # if no errors, process the primer primerFile.write('%d\t%s\t\t%d\t%d\t%s\t%s\t%s\t%s\t\t\t%s\t%s\t%s\t%s\t%s\n' \ % (primerKey, name, NA, vectorKey, segmentTypeKey, mgi_utils.prvalue(sequence1), \ mgi_utils.prvalue(sequence2), mgi_utils.prvalue(regionCovered), mgi_utils.prvalue(productSize), \ createdByKey, createdByKey, loaddate, loaddate)) for markerKey in markerList: if markerList.count(markerKey) == 1: markerFile.write('%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\n' \ % (primerKey, markerKey, referenceKey, relationship, createdByKey, createdByKey, loaddate, loaddate)) else: errorFile.write('Invalid Marker Duplicate: %s, %s\n' % (name, markerID)) # loaddate)) refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' % (refKey, primerKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate)) # aliases for alias in aliasList: if len(alias) == 0: continue aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ % (aliasKey, refKey, alias, createdByKey, createdByKey, loaddate, loaddate)) aliasKey = aliasKey + 1 # MGI Accession ID for the marker accFile.write('%s\t%s%d\t%s\t%s\t1\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \ % (accKey, mgiPrefix, mgiKey, mgiPrefix, mgiKey, primerKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate)) newPrimerFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%d\n' \ % (markerSymbol, string.join(markerIDs, '|'), name, jnum, regionCovered, sequence1, sequence2, productSize, notes, sequenceIDs, createdBy, mgiPrefix, mgiKey)) accKey = accKey + 1 mgiKey = mgiKey + 1 # sequence accession ids for acc in seqAccList: if len(acc) == 0: continue prefixPart, numericPart = accessionlib.split_accnum(acc) accFile.write('%s\t%s\t%s\t%s\t%s\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \ % (accKey, acc, prefixPart, numericPart, logicalDBKey, primerKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate)) accRefFile.write('%s\t%s\t%s\t%s\t%s\t%s\n' \ % (accKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate)) accKey = accKey + 1 # notes if len(notes) > 0: noteFile.write('%s|1\t%s\t%s\t%s\n' \ % (primerKey, notes, loaddate, loaddate)) refKey = refKey + 1 primerKey = primerKey + 1 # end of "for line in inputFile.readlines():" # # Update the AccessionMax value # if not DEBUG: db.sql('select * from ACC_setMax (%d)' % (lineNum), None)
def processFile(): global primerKey, refKey, aliasKey, accKey, mgiKey lineNum = 0 # For each line in the input file for line in inputFile.readlines(): error = 0 lineNum = lineNum + 1 # Split the line into tokens tokens = string.split(line[:-1], '\t') try: markerSymbol = tokens[0] # not used markerIDs = string.split(tokens[1], '|') name = tokens[2] jnum = tokens[3] regionCovered = tokens[4] sequence1 = tokens[5] sequence2 = tokens[6] productSize = tokens[7] notes = tokens[8] sequenceIDs = tokens[9] aliasList = string.split(tokens[10], '|') createdBy = tokens[11] except: exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line)) # marker IDs markerList = [] for markerID in markerIDs: markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile) if len(markerID) > 0 and markerKey == 0: errorFile.write('Invalid Marker: %s, %s\n' % (name, markerID)) error = 1 elif len(markerID) > 0: markerList.append(markerKey) referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile) createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile) # sequence IDs seqAccList = string.split(sequenceIDs, '|') # if errors, continue to next record if error: continue # if no errors, process the primer primerFile.write('%d\t%s\t\t%d\t%d\t%s\t%s\t%s\t%s\t\t\t%s\t%s\t%s\t%s\t%s\n' \ % (primerKey, name, NA, vectorKey, segmentTypeKey, mgi_utils.prvalue(sequence1), \ mgi_utils.prvalue(sequence2), mgi_utils.prvalue(regionCovered), mgi_utils.prvalue(productSize), \ createdByKey, createdByKey, loaddate, loaddate)) for markerKey in markerList: if markerList.count(markerKey) == 1: markerFile.write('%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\n' \ % (primerKey, markerKey, referenceKey, relationship, createdByKey, createdByKey, loaddate, loaddate)) else: errorFile.write('Invalid Marker Duplicate: %s, %s\n' % (name, markerID)) # loaddate)) refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' % (refKey, primerKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate)) # aliases for alias in aliasList: if len(alias) == 0: continue aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ % (aliasKey, refKey, alias, createdByKey, createdByKey, loaddate, loaddate)) aliasKey = aliasKey + 1 # MGI Accession ID for the marker accFile.write('%s\t%s%d\t%s\t%s\t1\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \ % (accKey, mgiPrefix, mgiKey, mgiPrefix, mgiKey, primerKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate)) newPrimerFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%d\n' \ % (markerSymbol, string.join(markerIDs, '|'), name, jnum, regionCovered, sequence1, sequence2, productSize, notes, sequenceIDs, createdBy, mgiPrefix, mgiKey)) accKey = accKey + 1 mgiKey = mgiKey + 1 # sequence accession ids for acc in seqAccList: if len(acc) == 0: continue prefixPart, numericPart = accessionlib.split_accnum(acc) accFile.write('%s\t%s\t%s\t%s\t%s\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \ % (accKey, acc, prefixPart, numericPart, logicalDBKey, primerKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate)) accRefFile.write('%s\t%s\t%s\t%s\t%s\t%s\n' \ % (accKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate)) accKey = accKey + 1 # notes if len(notes) > 0: noteFile.write('%s|1\t%s\t%s\t%s\n' \ % (primerKey, notes, loaddate, loaddate)) refKey = refKey + 1 primerKey = primerKey + 1 # end of "for line in inputFile.readlines():" # # Update the AccessionMax value # if not DEBUG: db.sql('select * from ACC_setMax (%d)' % (lineNum), None)
def process(expFile): global expCount, exptLoadedCount, updateExptCount, updateExptList global nextExptKey, nextAccKey, nextExptVarKey, nextPropKey global expSkippedNotInDbTransIsSuperseriesSet, expSkippedNoSampleList global expIdsInDbSet, expLoadedNoSampleList, expIdsInDbNoSamplesSet global expSkippedNotInDbNoTransSet, expSkippedMaxSamplesSet f = open(expFile, encoding='utf-8', errors='replace') context = ET.iterparse(f, events=("start", "end")) context = iter(context) level = 0 expID = '' title = '' summary = '' pdat = '' gdsType = '' exptType = '' n_samples = '' pubmedList = [] sampleList = [] # list of samplIDs isSuperSeries = 'no' # flag to indicate expt is superseries, skip exptTypeKey = 0 # if 0 chosen gdstype did not translate, skip isTpr = 0 try: for event, elem in context: # end of a record - reset everything if event == 'end' and elem.tag == 'DocumentSummary': expCount += 1 skip = 0 #print('\n\nexpID: %s' % expID) allExptIdList.append(expID) # # Experiment is in the database # add new pubmed ids # add raw sample data to those curated experiments that do not have it # if expID in primaryIdDict: updateExpKey = primaryIdDict[expID] # # check for additional pubmed IDs # propertyUpdateTemplate = "#====#%s%s%s#=#%s#=====#%s%s%s#==#%s#===#%s%s%s%s%s%s%s%s%s" % ( TAB, propTypeKey, TAB, TAB, TAB, exptMgiTypeKey, TAB, TAB, TAB, userKey, TAB, userKey, TAB, loadDate, TAB, loadDate, CRT) skip = 1 expIdsInDbSet.add(expID) #print(' expIdInDb skip') # not all experiments have pubmed IDs in the database # assigning empty list assures we pick up this case dbBibList = [] if expID in pubMedByExptDict: # get the list of pubmed Ids for this expt in the database dbBibList = pubMedByExptDict[expID] # get the set of incoming pubmed IDs not in the database newSet = set(pubmedList).difference(set(dbBibList)) # if we have new pubmed IDs, add them to the database if newSet: #print('found new pubmed ids: %s' % newSet) # get next sequenceNum for this expt's pubmed ID # in the database # get the next property sequence number results = db.sql( '''select max(sequenceNum) + 1 as nextNum from MGI_Property p where p._Object_key = %s and p._PropertyTerm_key = 20475430 and p._PropertyType_key = 1002''' % updateExpKey, 'auto') nextSeqNum = results[0]['nextNum'] if nextSeqNum == None: nextSeqNum = 1 updateExptCount += 1 updateExptList.append(expID) for b in newSet: toLoad = propertyUpdateTemplate.replace( '#=#', str(pubmedPropKey)).replace( '#==#', str(b)).replace( '#===#', str(nextSeqNum)).replace( '#====#', str(nextPropKey)).replace( '#=====#', str(updateExpKey)) fpPropertyBcp.write(toLoad) nextPropKey += 1 # if there's no raw sample data for the existing experiment, add it if expID in curatedGeoNoRawSampleDict: ret = processSamples(expID, 'true') # a list of sample info # means there was no sample file or 2 means there was # a parsing error if ret == 1 or ret == 2: print('expt inDb returnCode for %s: %s' % (expID, ret)) else: #print('expt inDb adding samples for key: %s expID: %s sampleInfo:%s' % (updateExpKey, expID, ret)) expIdsInDbNoSamplesSet.add(expID) # # GXD_HTRawSample and MGI_KeyValue BCP for # curated experiments (no raw sample data) # processSampleBcp(ret, updateExpKey) typeList = list(map(str.strip, gdsType.split(';'))) if skip != 1: (exptTypeKey, exptType) = processExperimentType(typeList) if exptTypeKey == 0: # expts whose type doesn't translate and is not already in the db expSkippedNotInDbNoTransSet.add(expID) skip = 1 #print(' expIdNotInDbNoTrans skip') if skip != 1 and isSuperSeries == 'yes': # number of superseries not already caught because of un translated # exptType or already in DB expSkippedNotInDbTransIsSuperseriesSet.add(expID) skip = 1 if skip != 1 and int(n_samples) > int(maxSamples): expSkippedMaxSamplesSet.add(expID) skip = 1 #print('exptTypeKey: %s isSuperSeries: %s skip: %s' % (exptTypeKey, isSuperSeries, skip)) if skip != 1: exptLoadedCount += 1 createExpObject = 0 # now process the samples ret = processSamples(expID, 'false') #print('ret: %s' % ret) if ret == 1: expLoadedNoSampleList.append('expID: %s' % (expID)) createExpObject = 1 elif ret == 2: expSkippedNoSampleList.append('expID: %s' % (expID)) exptLoadedCount -= 1 # decrement the loaded count else: sampleList = ret # list of sampleString's representing each # sample for the current experiment createExpObject = 1 if createExpObject: # catenate the global overallDesign parsed from the sample to the # experiment summary description = '%s %s' % (summary, overallDesign) description = description.replace('\t', ' ') description = description.replace('\n', ' ') if runParsingReports == 'true': fpExpParsingFile.write( '%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' % (expID, TAB, ', '.join(sampleList), TAB, title, TAB, description, TAB, isSuperSeries, TAB, pdat, TAB, exptType, TAB, n_samples, TAB, ', '.join(pubmedList), CRT)) # # GXD_HTExperiment BCP # line = '%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' % ( nextExptKey, TAB, sourceKey, TAB, title, TAB, description, TAB, pdat, TAB, releasedate, TAB, evalDate, TAB, evalStateTermKey, TAB, curStateTermKey, TAB, studyTypeTermKey, TAB, exptTypeKey, TAB, evalByKey, TAB, initCurByKey, TAB, lastCurByKey, TAB, initCurDate, TAB, lastCurDate, TAB, confidence, TAB, userKey, TAB, userKey, TAB, loadDate, TAB, loadDate, CRT) #print('line: %s' % line) fpExperimentBcp.write(line) # # GXD_HTVariable BCP # fpVariableBcp.write('%s%s%s%s%s%s' % (nextExptVarKey, TAB, nextExptKey, TAB, exptVariableTermKey, CRT)) nextExptVarKey += 1 # # ACC_Accession BCP # prefixPart, numericPart = accessionlib.split_accnum( expID) fpAccBcp.write( '%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' % (nextAccKey, TAB, expID, TAB, prefixPart, TAB, numericPart, TAB, geoLdbKey, TAB, nextExptKey, TAB, exptMgiTypeKey, TAB, private, TAB, isPreferred, TAB, userKey, TAB, userKey, TAB, loadDate, TAB, loadDate, CRT)) nextAccKey += 1 # # Experiment Properties # # title (1) experiment name # namePropKey = 20475428 # n_samples (1) count of samples # sampleCountPropKey = 20475424 # typeList (1-n) raw experiment types # expTypePropKey = 20475425 # pubmedList (0-n) pubmed Ids # pubmedPropKey = 20475430 # description (1) sample overalldesign + expt summary # descriptionPropKey = 87508020 # the template for properties: propertyTemplate = "#====#%s%s%s#=#%s%s%s%s%s#==#%s#===#%s%s%s%s%s%s%s%s%s" % ( TAB, propTypeKey, TAB, TAB, nextExptKey, TAB, exptMgiTypeKey, TAB, TAB, TAB, userKey, TAB, userKey, TAB, loadDate, TAB, loadDate, CRT) if title != '': toLoad = propertyTemplate.replace( '#=#', str(namePropKey)).replace( '#==#', title).replace('#===#', '1').replace( '#====#', str(nextPropKey)) fpPropertyBcp.write(toLoad) nextPropKey += 1 if n_samples != '': toLoad = propertyTemplate.replace( '#=#', str(sampleCountPropKey)).replace( '#==#', str(n_samples)).replace( '#===#', '1').replace('#====#', str(nextPropKey)) fpPropertyBcp.write(toLoad) nextPropKey += 1 seqNumCt = 1 for e in typeList: toLoad = propertyTemplate.replace( '#=#', str(expTypePropKey)).replace( '#==#', e).replace('#===#', str(seqNumCt)).replace( '#====#', str(nextPropKey)) fpPropertyBcp.write(toLoad) seqNumCt += 1 nextPropKey += 1 for b in pubmedList: toLoad = propertyTemplate.replace( '#=#', str(pubmedPropKey)).replace( '#==#', str(b)).replace('#===#', str(seqNumCt)).replace( '#====#', str(nextPropKey)) fpPropertyBcp.write(toLoad) seqNumCt += 1 nextPropKey += 1 if title != '': toLoad = propertyTemplate.replace( '#=#', str(namePropKey)).replace( '#==#', title).replace('#===#', '1').replace( '#====#', str(nextPropKey)) fpPropertyBcp.write(toLoad) nextPropKey += 1 # # GXD_HTRawSample and MGI_KeyValue BCP # # ret from processSample = 1 means there was no sample file # so exeriment is created, but no samples if ret != 1: processSampleBcp(sampleList, nextExptKey) # now increment the experiment key nextExptKey += 1 title = '' summary = '' isSuperSeries = 'no' pdat = '' gdsType = '' n_samples = '' pubmedList = [] sampleList = [] exptTypeKey = 0 if level == 4: # Accession tag at level 4 tells us we have a new record if elem.tag == 'Accession': expID = elem.text elif elem.tag == 'title': title = elem.text elif elem.tag == 'summary': summary = elem.text if summary.find(SUPERSERIES) != -1: isSuperSeries = 'yes' #print('isSuperSeries: %s' % expID) elif elem.tag == 'PDAT': pdat = elem.text elif elem.tag == 'gdsType': gdsType = elem.text elif elem.tag == 'n_samples': n_samples = elem.text if event == 'start': level += 1 #print('level: %s elemTag: %s elemText: %s' % (level, elem.tag, elem.text)) elif elem.tag == 'int': id = elem.text #print('id: %s' % id) pubmedList.append(id) elif level == 6 and elem.tag == 'Accession': sampleList.append(elem.text) if event == 'end': level -= 1 elem.clear() except: print('Parsing error: on %s' % expFile) return 1 return 0
def process(): """ Query database to determine if dummy sequences need to be created Generates appropriate BCP files """ global seqKey, assocKey, accKey # generate table of all mouse molecular segments Acc IDs whose GenBank SeqIDs # are not represented as Sequence objects. db.sql("""select a.accID, a._LogicalDB_key, ps._Organism_key INTO TEMPORARY TABLE probeaccs1 from ACC_Accession a, PRB_Probe p, PRB_Source ps where a._MGIType_key = 3 and a._LogicalDB_key = 9 and a._Object_key = p._Probe_key and p._Source_key = ps._Source_key and ps._Organism_key = 1 and not exists (select 1 from ACC_Accession s where s._MGIType_key = 19 and s._LogicalDB_key = a._LogicalDB_key and lower(s.accID) = lower(a.accID) )""", None) # generate table of all mouse marker Acc IDs whose GenBank, SWISSProt, RefSeq, # TrEMBL IDs are not represented as Sequence objects. db.sql("""select a.accID, a._LogicalDB_key, m._Organism_key INTO TEMPORARY TABLE markeraccs1 from ACC_Accession a, MRK_Marker m where a._MGIType_key = 2 and a._LogicalDB_key in (9,13,27,41) and a._Object_key = m._Marker_key and m._Organism_key = 1 and m._Marker_Status_key in (1,2) and not exists (select 1 from ACC_Accession s where s._MGIType_key = 19 and s._LogicalDB_key = a._LogicalDB_key and lower(s.accID) = lower(a.accID) )""", None) # generate table of all non-mouse molecular segments Acc IDs whose GenBank SeqIDs # are not represented as Sequence objects. db.sql("""select a.accID, a._LogicalDB_key, s._Organism_key INTO TEMPORARY TABLE probeaccs2 from ACC_Accession a, PRB_Probe p, PRB_Source s where a._MGIType_key = 3 and a._LogicalDB_key = 9 and a._Object_key = p._Probe_key and p._Source_key = s._Source_key and s._Organism_key != 1 and not exists (select 1 from ACC_Accession s where s._MGIType_key = 19 and s._LogicalDB_key = a._LogicalDB_key and lower(s.accID) = lower(a.accID) )""", None) # generate table of all non-mouse marker Acc IDs whose GenBank, SWISSProt, RefSeq, # TrEMBL IDs are not represented as Sequence objects. db.sql("""select a.accID, a._LogicalDB_key, m._Organism_key INTO TEMPORARY TABLE markeraccs2 from ACC_Accession a, MRK_Marker m where a._MGIType_key = 2 and a._LogicalDB_key in (9,13,27,41) and a._Object_key = m._Marker_key and m._Organism_key != 1 and not exists (select 1 from ACC_Accession s where s._MGIType_key = 19 and s._LogicalDB_key = a._LogicalDB_key and lower(s.accID) = lower(a.accID) )""", None) # union these 4 sets together to form one unique set db.sql('select accID, _LogicalDB_key, _Organism_key ' + \ 'INTO TEMPORARY TABLE allaccs ' + \ 'from probeaccs1 ' + \ 'union ' + \ 'select accID, _LogicalDB_key, _Organism_key ' + \ 'from markeraccs1 ' + \ 'union ' + \ 'select accID, _LogicalDB_key, _Organism_key ' + \ 'from probeaccs2 ' + \ 'union ' + \ 'select accID, _LogicalDB_key, _Organism_key ' + \ 'from markeraccs2', None) results = db.sql('select * from allaccs', 'auto') for r in results: accID = r['accID'] logicalDB = r['_LogicalDB_key'] organism = r['_Organism_key'] if organism == 1: sourceKey = mouseSourceKey else: sourceKey = nonmouseSourceKey virtual = 1 # change values for specific cases # types: 316347 (DNA), 316346 (RNA), 316348 (polypeptide), 316349 (not loaded) # quality: 316338 (high), 316339 (medium), 316340 (low), 316341 (not loaded) # provider: 316380 (GenBank/EMBL/DDBJ), 316372 (RefSeq) # 316384 (SwissProt), 316385 (TrEMBL) if logicalDB == 9: # GenBank typeKey = 316349 qualityKey = 316341 providerKey = 316380 virtual = 0 elif logicalDB == 27: # RefSeq typeKey = 316349 qualityKey = 316338 providerKey = 316372 elif logicalDB == 13: # SwissProt typeKey = 316348 qualityKey = 316338 providerKey = 316384 elif logicalDB == 41: # TrEMBL typeKey = 316348 qualityKey = 316340 providerKey = 316385 seqFile.write(mgi_utils.prvalue(seqKey) + DL + \ mgi_utils.prvalue(typeKey) + DL + \ mgi_utils.prvalue(qualityKey) + DL + \ mgi_utils.prvalue(statusKey) + DL + \ mgi_utils.prvalue(providerKey) + DL + \ mgi_utils.prvalue(organism) + DL + \ DL + DL + DL + DL + \ mgi_utils.prvalue(virtual) + DL + \ DL + \ loaddate + DL + loaddate + DL + \ str(userKey) + DL + str(userKey) + DL + \ loaddate + DL + loaddate + NL) rawFile.write(mgi_utils.prvalue(seqKey) + DL + \ notLoaded + DL + \ notLoaded + DL + \ notLoaded + DL + \ notLoaded + DL + \ notLoaded + DL + \ notLoaded + DL + \ notLoaded + DL + \ notLoaded + DL + \ str(userKey) + DL + str(userKey) + DL + \ loaddate + DL + loaddate + NL) sourceFile.write(mgi_utils.prvalue(assocKey) + DL + \ mgi_utils.prvalue(seqKey) + DL + \ mgi_utils.prvalue(sourceKey) + DL + \ str(userKey) + DL + str(userKey) + DL + \ loaddate + DL + loaddate + NL) prefixPart, numericPart = accessionlib.split_accnum(accID) accFile.write(mgi_utils.prvalue(accKey) + DL + \ mgi_utils.prvalue(accID) + DL + \ mgi_utils.prvalue(prefixPart) + DL + \ mgi_utils.prvalue(numericPart) + DL + \ mgi_utils.prvalue(logicalDB) + DL + \ mgi_utils.prvalue(seqKey) + DL + \ mgi_utils.prvalue(mgiTypeKey) + DL + \ '0' + DL + \ '1' + DL + \ str(userKey) + DL + str(userKey) + DL + \ loaddate + DL + loaddate + NL) seqKey = seqKey + 1 assocKey = assocKey + 1 accKey = accKey + 1
def processFile(): global probeKey, refKey, aliasKey, accKey, mgiKey lineNum = 0 # For each line in the input file for line in inputFile.readlines(): error = 0 lineNum = lineNum + 1 # Split the line into tokens tokens = string.split(line[:-1], '\t') try: name = tokens[0] jnum = tokens[1] parentID = tokens[2] sourceName = tokens[3] organism = tokens[4] strain = tokens[5] tissue = tokens[6] gender = tokens[7] cellLine = tokens[8] age = tokens[9] vectorType = tokens[10] segmentType = tokens[11] regionCovered = tokens[12] insertSite = tokens[13] insertSize = tokens[14] markerIDs = string.split(tokens[15], '|') relationship = tokens[16] sequenceIDs = tokens[17] aliasList = string.split(tokens[18], '|') notes = tokens[19] rawnotes = tokens[20] createdBy = tokens[21] except: exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line)) isParent = 0 isSource = 0 parentProbeKey = ''; sourceKey = 0 if parentID != '': isParent = 1 if sourceName != '': isSource = 1 if not isParent and not isSource: organismKey = sourceloadlib.verifyOrganism(organism, lineNum, errorFile) strainKey = sourceloadlib.verifyStrain(strain, lineNum, errorFile) tissueKey = sourceloadlib.verifyTissue(tissue, lineNum, errorFile) genderKey = sourceloadlib.verifyGender(gender, lineNum, errorFile) cellLineKey = sourceloadlib.verifyCellLine(cellLine, lineNum, errorFile) vectorKey = sourceloadlib.verifyVectorType(vectorType, lineNum, errorFile) segmentTypeKey = sourceloadlib.verifySegmentType(segmentType, lineNum, errorFile) sourceKey = sourceloadlib.verifySource(segmentTypeKey, \ vectorKey, organismKey, strainKey, \ tissueKey, genderKey, cellLineKey, age, lineNum, errorFile) if organismKey == 0 or strainKey == 0 or tissueKey == 0 or \ genderKey == 0 or cellLineKey == 0 or vectorKey == 0 or \ segmentTypeKey == 0 or sourceKey == 0: errorFile.write('%s, %s, %s, %s, %s, %s, %s, %s\n' % (segmentType, vectorType, organism, strain, tissue, gender, cellLine, age)) error = 1 elif not isParent and isSource: vectorKey = sourceloadlib.verifyVectorType(vectorType, lineNum, errorFile) segmentTypeKey = sourceloadlib.verifySegmentType(segmentType, lineNum, errorFile) sourceKey = sourceloadlib.verifyLibrary(sourceName, lineNum, errorFile) if vectorKey == 0 or segmentTypeKey == 0 or sourceKey == 0: error = 1 # parent from = yes, source given = yes or no (ignored) else: parentProbeKey, sourceKey = verifyParentProbe(parentID, lineNum, errorFile) vectorKey = sourceloadlib.verifyVectorType(vectorType, lineNum, errorFile) segmentTypeKey = sourceloadlib.verifySegmentType(segmentType, lineNum, errorFile) if parentProbeKey == 0 or sourceKey == 0 or vectorKey == 0 or segmentTypeKey == 0: error = 1 referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile) createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile) if referenceKey == 0: errorFile.write('Invalid Reference: %s\n' % (jnum)) error = 1 if createdByKey == 0: errorFile.write('Invalid Creator: %s\n\n' % (createdBy)) error = 1 # marker IDs markerList = [] for markerID in markerIDs: markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile) if len(markerID) > 0 and markerKey == 0: errorFile.write('Invalid Marker: %s, %s\n' % (name, markerID)) error = 1 elif len(markerID) > 0: markerList.append(markerKey) # sequence IDs seqAccDict = {} for seqID in string.split(sequenceIDs, '|'): if len(seqID) > 0: [logicalDB, acc] = string.split(seqID, ':') logicalDBKey = loadlib.verifyLogicalDB(logicalDB, lineNum, errorFile) if logicalDBKey > 0: seqAccDict[acc] = logicalDBKey # if errors, continue to next record if error: continue # if no errors, process the probe probeFile.write('%d\t%s\t%s\t%s\t%s\t%s\t\t\t%s\t%s\t%s\t\t%s\t%s\t%s\t%s\n' \ % (probeKey, name, parentProbeKey, sourceKey, vectorKey, segmentTypeKey, mgi_utils.prvalue(regionCovered), \ mgi_utils.prvalue(insertSite), mgi_utils.prvalue(insertSize), createdByKey, createdByKey, loaddate, loaddate)) for markerKey in markerList: if markerList.count(markerKey) == 1: markerFile.write('%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\n' \ % (probeKey, markerKey, referenceKey, relationship, createdByKey, createdByKey, loaddate, loaddate)) else: errorFile.write('Invalid Marker Duplicate: %s, %s\n' % (name, markerID)) refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' \ % (refKey, probeKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate)) # aliases for alias in aliasList: if len(alias) == 0: continue aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ % (aliasKey, refKey, alias, createdByKey, createdByKey, loaddate, loaddate)) aliasKey = aliasKey + 1 # MGI Accession ID for the marker accFile.write('%s\t%s%d\t%s\t%s\t1\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \ % (accKey, mgiPrefix, mgiKey, mgiPrefix, mgiKey, probeKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate)) # Print out a new text file and attach the new MGI Probe IDs as the last field newProbeFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%d\n' \ % (name, jnum, \ mgi_utils.prvalue(sourceName), \ organism, \ mgi_utils.prvalue(strain), \ mgi_utils.prvalue(tissue), \ mgi_utils.prvalue(gender), \ mgi_utils.prvalue(cellLine), \ mgi_utils.prvalue(age), \ mgi_utils.prvalue(vectorType), \ mgi_utils.prvalue(segmentType), \ mgi_utils.prvalue(regionCovered) + \ mgi_utils.prvalue(insertSite), \ mgi_utils.prvalue(insertSize), \ string.join(markerIDs, '|'), \ relationship, \ mgi_utils.prvalue(sequenceIDs), \ string.join(aliasList, '|'), \ mgi_utils.prvalue(notes), \ createdBy, mgiPrefix, mgiKey)) # Print out a raw note file if len(rawnotes) > 0: rawNoteFile.write('%s%d\t%s\n' % (mgiPrefix, mgiKey, rawnotes)) # Notes if len(notes) > 0: noteFile.write('%s\t%s\t%s\t%s\n' % (probeKey, notes, loaddate, loaddate)) accKey = accKey + 1 mgiKey = mgiKey + 1 # sequence accession ids for acc in seqAccDict.keys(): prefixPart, numericPart = accessionlib.split_accnum(acc) accFile.write('%s\t%s\t%s\t%s\t%s\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \ % (accKey, acc, prefixPart, numericPart, seqAccDict[acc], probeKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate)) accRefFile.write('%s\t%s\t%s\t%s\t%s\t%s\n' \ % (accKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate)) accKey = accKey + 1 refKey = refKey + 1 probeKey = probeKey + 1 # end of "for line in inputFile.readlines():" # # Update the AccessionMax value # if not DEBUG: db.sql('select * from ACC_setMax (%d)' % (lineNum), None)
def writeB6Output(): # Purpose: parses the output line dictionary # writes to Accession, AccessionReference & StrainMarker BCP file # if there are no errors # Returns: 1 if error, else 0 # Assumes: file descriptors have been initialized # Effects: writes to the file system # Throws: Nothing global nextSMKey, nextAccKey, totalLoadedCt, b6LoadedCt description = '' for mgiID in b6ToLoadDict: #print 'writeB6Output mgiID: "%s"' % mgiID lineList = b6ToLoadDict[mgiID] #print 'writeB6Output lineList: %s' % lineList qNameSet = set() # Resolve MGI ID if mgiID not in markerLookup: print('%s in MGI GFF File, but NOT IN MGI' % (mgiID)) continue else: marker = markerLookup[mgiID] markerKey = marker.markerKey symbol = marker.symbol if len(lineList) == 1: # This is non-BlatAlignment gene/pseudogene #print 'This is non-BlatAlignment gene/pseudogene and nextSMKey: %s' % nextSMKey line = lineList[0] chr, start, end, strand, smID, mgiID, biotype, gmIdString, qName, description = parseB6Feature( line, 'f') fpStrainMarkerFile.write( '%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' % (nextSMKey, TAB, b6StrainKey, TAB, markerKey, TAB, b6RefsKey, TAB, userKey, TAB, userKey, TAB, loaddate, TAB, loaddate, CRT)) prefixPart, numericPart = accessionlib.split_accnum(smID) fpAccFile.write('%s%s%s%s%s%s%s%s%s%s%s%s%s%s0%s1%s%s%s%s%s%s%s%s%s' \ % (nextAccKey, TAB, smID, TAB, prefixPart, TAB, numericPart, TAB, msgLDBKey, TAB, nextSMKey, TAB, mgiTypeKey, TAB, TAB, TAB, userKey, TAB, userKey, TAB, loaddate, TAB, loaddate, CRT)) fpAccRefFile.write('%s%s%s%s%s%s%s%s%s%s%s%s' \ % (nextAccKey, TAB, b6RefsKey, TAB, userKey, TAB, userKey, TAB, loaddate, TAB, loaddate, CRT)) fpGmB6File.write('%s%s%s%s%s%s%s%s%s%s%s%s' % (smID, TAB, chr, TAB, start, TAB, end, TAB, strand, TAB, description, CRT)) fpBiotypeB6File.write('%s%s%s%s' % (smID, TAB, biotype, CRT)) nextAccKey += 1 # get gmIDs from the input file for the sequence description, if they exist # gmIDs example: # Dbxref=miRBase:MI0005004,ENSEMBL:ENSMUSG00000076010,NCBI_Gene:751557 if gmIdString == '': continue gmIdList = gmIdString.split(',') nextSMKey += 1 else: # This is BlatAlignment set #print 'this is a BlatAlignment set nextSMKey: %s' % nextSMKey # The first line is feature line, the following are BlatAlignments featureLine = lineList[0] chr, start, end, strand, smID, mgiID, biotype, gmIdString, qName, description = parseB6Feature( featureLine, 'bf') # remainder of the list are blat hits - save them to a set lineList = lineList[1:] # remove the blat feature for line in lineList: j1, j2, j3, j4, j5, j6, j7, j8, qName, j9 = parseB6Feature( line, 'b') qNameSet.add(qName.strip()) # set the qNames (genbank IDs) in the description string description = description % ','.join(str(s) for s in qNameSet) # # Create the strain marker and its accession ID # fpStrainMarkerFile.write( '%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' % (nextSMKey, TAB, b6StrainKey, TAB, markerKey, TAB, b6RefsKey, TAB, userKey, TAB, userKey, TAB, loaddate, TAB, loaddate, CRT)) prefixPart, numericPart = accessionlib.split_accnum(smID) fpAccFile.write('%s%s%s%s%s%s%s%s%s%s%s%s%s%s0%s1%s%s%s%s%s%s%s%s%s' \ % (nextAccKey, TAB, smID, TAB, prefixPart, TAB, numericPart, TAB, msgLDBKey, TAB, nextSMKey, TAB, mgiTypeKey, TAB, TAB, TAB, userKey, TAB, userKey, TAB, loaddate, TAB, loaddate, CRT)) fpAccRefFile.write('%s%s%s%s%s%s%s%s%s%s%s%s' \ % (nextAccKey, TAB, b6RefsKey, TAB, userKey, TAB, userKey, TAB, loaddate, TAB, loaddate, CRT)) nextAccKey += 1 fpGmB6File.write('%s%s%s%s%s%s%s%s%s%s%s%s' % (smID, TAB, chr, TAB, start, TAB, end, TAB, strand, TAB, description, CRT)) fpBiotypeB6File.write('%s%s%s%s' % (smID, TAB, biotype, CRT)) # 6/12 GF-184, removed all associated IDs from strain gene # for blat hits in the input file associate the GenBank IDs that was # blatted for coordinates nextSMKey += 1 totalLoadedCt += 1 b6LoadedCt += 1 return 0
def processFile(): global probeKey, refKey, aliasKey, accKey, mgiKey lineNum = 0 # For each line in the input file for line in inputFile.readlines(): error = 0 lineNum = lineNum + 1 # Split the line into tokens tokens = string.split(line[:-1], '\t') try: name = tokens[0] jnum = tokens[1] parentID = tokens[2] sourceName = tokens[3] organism = tokens[4] strain = tokens[5] tissue = tokens[6] gender = tokens[7] cellLine = tokens[8] age = tokens[9] vectorType = tokens[10] segmentType = tokens[11] regionCovered = tokens[12] insertSite = tokens[13] insertSize = tokens[14] markerIDs = string.split(tokens[15], '|') relationship = tokens[16] sequenceIDs = tokens[17] aliasList = string.split(tokens[18], '|') notes = tokens[19] rawnotes = tokens[20] createdBy = tokens[21] except: exit(1, 'Invalid Line (%d): %s\n' % (lineNum, line)) isParent = 0 isSource = 0 parentProbeKey = '' sourceKey = 0 if parentID != '': isParent = 1 if sourceName != '': isSource = 1 if not isParent and not isSource: organismKey = sourceloadlib.verifyOrganism(organism, lineNum, errorFile) strainKey = sourceloadlib.verifyStrain(strain, lineNum, errorFile) tissueKey = sourceloadlib.verifyTissue(tissue, lineNum, errorFile) genderKey = sourceloadlib.verifyGender(gender, lineNum, errorFile) cellLineKey = sourceloadlib.verifyCellLine(cellLine, lineNum, errorFile) vectorKey = sourceloadlib.verifyVectorType(vectorType, lineNum, errorFile) segmentTypeKey = sourceloadlib.verifySegmentType( segmentType, lineNum, errorFile) sourceKey = sourceloadlib.verifySource(segmentTypeKey, \ vectorKey, organismKey, strainKey, \ tissueKey, genderKey, cellLineKey, age, lineNum, errorFile) if organismKey == 0 or strainKey == 0 or tissueKey == 0 or \ genderKey == 0 or cellLineKey == 0 or vectorKey == 0 or \ segmentTypeKey == 0 or sourceKey == 0: errorFile.write('%s, %s, %s, %s, %s, %s, %s, %s\n' % (segmentType, vectorType, organism, strain, tissue, gender, cellLine, age)) error = 1 elif not isParent and isSource: vectorKey = sourceloadlib.verifyVectorType(vectorType, lineNum, errorFile) segmentTypeKey = sourceloadlib.verifySegmentType( segmentType, lineNum, errorFile) sourceKey = sourceloadlib.verifyLibrary(sourceName, lineNum, errorFile) if vectorKey == 0 or segmentTypeKey == 0 or sourceKey == 0: error = 1 # parent from = yes, source given = yes or no (ignored) else: parentProbeKey, sourceKey = verifyParentProbe( parentID, lineNum, errorFile) vectorKey = sourceloadlib.verifyVectorType(vectorType, lineNum, errorFile) segmentTypeKey = sourceloadlib.verifySegmentType( segmentType, lineNum, errorFile) if parentProbeKey == 0 or sourceKey == 0 or vectorKey == 0 or segmentTypeKey == 0: error = 1 referenceKey = loadlib.verifyReference(jnum, lineNum, errorFile) createdByKey = loadlib.verifyUser(createdBy, lineNum, errorFile) if referenceKey == 0: errorFile.write('Invalid Reference: %s\n' % (jnum)) error = 1 if createdByKey == 0: errorFile.write('Invalid Creator: %s\n\n' % (createdBy)) error = 1 # marker IDs markerList = [] for markerID in markerIDs: markerKey = loadlib.verifyMarker(markerID, lineNum, errorFile) if len(markerID) > 0 and markerKey == 0: errorFile.write('Invalid Marker: %s, %s\n' % (name, markerID)) error = 1 elif len(markerID) > 0: markerList.append(markerKey) # sequence IDs seqAccDict = {} for seqID in string.split(sequenceIDs, '|'): if len(seqID) > 0: [logicalDB, acc] = string.split(seqID, ':') logicalDBKey = loadlib.verifyLogicalDB(logicalDB, lineNum, errorFile) if logicalDBKey > 0: seqAccDict[acc] = logicalDBKey # if errors, continue to next record if error: continue # if no errors, process the probe probeFile.write('%d\t%s\t%s\t%s\t%s\t%s\t\t\t%s\t%s\t%s\t\t%s\t%s\t%s\t%s\n' \ % (probeKey, name, parentProbeKey, sourceKey, vectorKey, segmentTypeKey, mgi_utils.prvalue(regionCovered), \ mgi_utils.prvalue(insertSite), mgi_utils.prvalue(insertSize), createdByKey, createdByKey, loaddate, loaddate)) for markerKey in markerList: if markerList.count(markerKey) == 1: markerFile.write('%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\n' \ % (probeKey, markerKey, referenceKey, relationship, createdByKey, createdByKey, loaddate, loaddate)) else: errorFile.write('Invalid Marker Duplicate: %s, %s\n' % (name, markerID)) refFile.write('%s\t%s\t%s\t0\t0\t%s\t%s\t%s\t%s\n' \ % (refKey, probeKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate)) # aliases for alias in aliasList: if len(alias) == 0: continue aliasFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ % (aliasKey, refKey, alias, createdByKey, createdByKey, loaddate, loaddate)) aliasKey = aliasKey + 1 # MGI Accession ID for the marker accFile.write('%s\t%s%d\t%s\t%s\t1\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \ % (accKey, mgiPrefix, mgiKey, mgiPrefix, mgiKey, probeKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate)) # Print out a new text file and attach the new MGI Probe IDs as the last field newProbeFile.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%d\n' \ % (name, jnum, \ mgi_utils.prvalue(sourceName), \ organism, \ mgi_utils.prvalue(strain), \ mgi_utils.prvalue(tissue), \ mgi_utils.prvalue(gender), \ mgi_utils.prvalue(cellLine), \ mgi_utils.prvalue(age), \ mgi_utils.prvalue(vectorType), \ mgi_utils.prvalue(segmentType), \ mgi_utils.prvalue(regionCovered) + \ mgi_utils.prvalue(insertSite), \ mgi_utils.prvalue(insertSize), \ string.join(markerIDs, '|'), \ relationship, \ mgi_utils.prvalue(sequenceIDs), \ string.join(aliasList, '|'), \ mgi_utils.prvalue(notes), \ createdBy, mgiPrefix, mgiKey)) # Print out a raw note file if len(rawnotes) > 0: rawNoteFile.write('%s%d\t%s\n' % (mgiPrefix, mgiKey, rawnotes)) # Notes if len(notes) > 0: noteFile.write('%s\t%s\t%s\t%s\n' % (probeKey, notes, loaddate, loaddate)) accKey = accKey + 1 mgiKey = mgiKey + 1 # sequence accession ids for acc in seqAccDict.keys(): prefixPart, numericPart = accessionlib.split_accnum(acc) accFile.write('%s\t%s\t%s\t%s\t%s\t%d\t%d\t0\t1\t%s\t%s\t%s\t%s\n' \ % (accKey, acc, prefixPart, numericPart, seqAccDict[acc], probeKey, mgiTypeKey, createdByKey, createdByKey, loaddate, loaddate)) accRefFile.write('%s\t%s\t%s\t%s\t%s\t%s\n' \ % (accKey, referenceKey, createdByKey, createdByKey, loaddate, loaddate)) accKey = accKey + 1 refKey = refKey + 1 probeKey = probeKey + 1 # end of "for line in inputFile.readlines():" # # Update the AccessionMax value # if not DEBUG: db.sql('select * from ACC_setMax (%d)' % (lineNum), None)