def exit(
# Purpose: prints error 'message' if it is not None
# writes to log files and exits with 'status'
# Returns: nothing
# Assumes: Nothing
# Effects: Exits with 'status'
status, # numeric exit status (integer)
message=None # exit message (str.
):
if message is not None:
sys.stderr.write('\n' + str(message) + '\n')
try:
fpDiagFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
fpErrorFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
fpDiagFile.close()
fpErrorFile.close()
fpInputFile.close()
except:
pass
db.useOneConnection(0)
sys.exit(status)
def exit(status, message=None):
'''
# requires: status, the numeric exit status (integer)
# message (string)
#
# effects:
# Print message to stderr and exits
#
# returns:
#
'''
if message is not None:
sys.stderr.write('\n' + str(message) + '\n')
try:
inputFile.close()
diagFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
errorFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
diagFile.close()
errorFile.close()
transTypeFile.close()
transFile.close()
except:
pass
db.useOneConnection()
sys.exit(status)
def exit(status, message = None):
'''
# requires: status, the numeric exit status (integer)
# message (string)
#
# effects:
# Print message to stderr and exits
#
# returns:
#
'''
if message is not None:
sys.stderr.write('\n' + str(message) + '\n')
try:
inputFile.close()
outputFile.close()
diagFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
errorFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
diagFile.close()
errorFile.close()
except:
pass
db.useOneConnection()
sys.exit(status)
def init():
#
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes global file descriptors/file names
#
# returns:
#
global diagFile, errorFile
db.useOneConnection(1)
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
# Set Log File Descriptor
db.set_sqlLogFD(diagFile)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
def init():
#
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes global file descriptors/file names
#
# returns:
#
global diagFile, errorFile, diagFileName, errorFileName
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
errorFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
def exit(
status, # numeric exit status (integer)
message=None # exit message (string)
):
# Purpose: writes message to error log and exits
# Returns: nothing
# Assumes: nothing
# Effects: nothing
# Throws: nothing
if message is not None:
sys.stderr.write('\n' + str(message) + '\n')
try:
inputFile.close()
diagFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
errorFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
diagFile.close()
errorFile.close()
except:
pass
db.useOneConnection(0)
sys.exit(status)
def init():
'''
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes local DBMS parameters
# 3. Initializes global file descriptors/file names
# 4. Initializes global keys
#
# returns:
#
'''
global inputFile, outputFile, diagFile, errorFile, errorFileName, diagFileName
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
diagFileName = inputFileName + '.diagnostics'
errorFileName = inputFileName + '.error'
outputFileName = inputFileName + '.trans'
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
outputFile = open(outputFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % outputFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
# Set Log File Descriptor
db.set_sqlLogFD(diagFile)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
diagFile.write('Input File: %s\n' % (inputFileName))
diagFile.write('Output File: %s\n' % (outputFileName))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
def init():
global bcpCommand
global diagFile, errorFile, inputFile, errorFileName, diagFileName
global markerFile, refFile, aliasFile
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
bcpCommand = bcpCommand + db.get_sqlServer() + ' ' + db.get_sqlDatabase(
) + ' %s ' + currentDir + ' %s "\\t" "\\n" mgd'
head, tail = os.path.split(inputFileName)
diagFileName = outputDir + '/' + tail + '.diagnostics'
errorFileName = outputDir + '/' + tail + '.error'
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
markerFile = open(markerFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % markerFileName)
try:
refFile = open(refFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % refFileName)
try:
aliasFile = open(aliasFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % aliasFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
return
def init():
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes local DBMS parameters
# 3. Initializes global file descriptors/file names
# 4. Initializes global keys
#
# returns:
#
global diagFile, errorFile, inputFile, errorFileName, diagFileName
global strainFile
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
fdate = mgi_utils.date('%m%d%Y') # current date
head, tail = os.path.split(inputFileName)
diagFileName = tail + '.' + fdate + '.diagnostics'
errorFileName = tail + '.' + fdate + '.error'
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
strainFile = open(strainFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % strainFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
# Set Log File Descriptor
db.set_sqlLogFD(diagFile)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
return
def init():
'''
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes local DBMS parameters
# 3. Initializes global file descriptors/file names
# 4. Initializes global keys
#
# returns:
#
'''
global inputFile, outputFile, diagFile, errorFile, errorFileName, diagFileName
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
diagFileName = inputFileName + '.diagnostics'
errorFileName = inputFileName + '.error'
outputFileName = inputFileName + '.trans'
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
outputFile = open(outputFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % outputFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
# Set Log File Descriptor
db.set_sqlLogFD(diagFile)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
diagFile.write('Input File: %s\n' % (inputFileName))
diagFile.write('Output File: %s\n' % (outputFileName))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
def init():
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes local DBMS parameters
# 3. Initializes global file descriptors/file names
# 4. Initializes global keys
#
# returns:
#
global diagFile, errorFile, inputFile, errorFileName, diagFileName
global strainFile
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
fdate = mgi_utils.date('%m%d%Y') # current date
head, tail = os.path.split(inputFileName)
diagFileName = tail + '.' + fdate + '.diagnostics'
errorFileName = tail + '.' + fdate + '.error'
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
strainFile = open(strainFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % strainFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
# Set Log File Descriptor
db.set_sqlLogFD(diagFile)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
return
def init():
global bcpCommand
global diagFile, errorFile, inputFile, errorFileName, diagFileName
global markerFile, refFile, aliasFile
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
bcpCommand = bcpCommand + db.get_sqlServer() + ' ' + db.get_sqlDatabase() + ' %s ' + currentDir + ' %s "\\t" "\\n" mgd'
head, tail = os.path.split(inputFileName)
diagFileName = outputDir + '/' + tail + '.diagnostics'
errorFileName = outputDir + '/' + tail + '.error'
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
markerFile = open(markerFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % markerFileName)
try:
refFile = open(refFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % refFileName)
try:
aliasFile = open(aliasFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % aliasFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
return
def createExcluded():
excludeNote = 'The source of the material used to create this cDNA probe was different than that used to create the GenBank sequence record.'
print 'excluded begin...%s' % (mgi_utils.date())
db.sql('''select _Probe_key INTO TEMPORARY TABLE excluded from PRB_Notes
where note like 'The source of the material used to create this cDNA probe was different%'
''', None)
db.sql('create index idx1 on excluded(_Probe_key)', None)
print 'excluded end...%s' % (mgi_utils.date())
def init():
'''
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes local DBMS parameters
# 3. Initializes global file descriptors/file names
#
# returns:
#
'''
global inputFile, outputFile, diagFile, errorFile
global inputFileName, errorFileName, diagFileName, outputFileName
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
diagFile = open(diagFileName, 'w+')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
outputFile = open(outputFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % outputFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
# Set Log File Descriptor
db.set_commandLogFile(diagFileName)
# Set Log File Descriptor
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
diagFile.write('Input File: %s\n' % (inputFileName))
errorFile.write('\nStart file: %s\n\n' % (mgi_utils.date()))
def init():
# Purpose: process command line options
# Returns: nothing
# Assumes: nothing
# Effects: initializes global variables
# exits if files cannot be opened
# Throws: nothing
global inputFile, diagFile, errorFile, errorFileName, diagFileName
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
fdate = mgi_utils.date('%m%d%Y') # current date
head, tail = os.path.split(inputFileName)
diagFileName = tail + '.' + fdate + '.diagnostics'
errorFileName = tail + '.' + fdate + '.error'
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
# Set Log File Descriptor
db.set_sqlLogFD(diagFile)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
diagFile.write('Input File: %s\n' % (inputFileName))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
return
def init():
'''
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes local DBMS parameters
# 3. Initializes global file descriptors/file names
#
# returns:
#
'''
global inputFile, outputFile, diagFile, errorFile
global inputFileName, errorFileName, diagFileName, outputFileName
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
diagFile = open(diagFileName, 'w+')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
outputFile = open(outputFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % outputFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
# Set Log File Descriptor
db.set_commandLogFile(diagFileName)
# Set Log File Descriptor
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
diagFile.write('Input File: %s\n' % (inputFileName))
errorFile.write('\nStart file: %s\n\n' % (mgi_utils.date()))
def exit(status, message=None):
'''
# requires: status, the numeric exit status (integer)
# message (string)
#
# effects:
# Print message to stderr and exits
#
# returns:
#
'''
if message is not None:
sys.stderr.write('\n' + str(message) + '\n')
try:
diagFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
diagFile.close()
accFile.close()
accrefFile.close()
markerFile.close()
except:
pass
sys.exit(status)
def init():
# Purpose: initialize file descriptors, create connection to
# the database and load lookups
# Returns: nothing
# Assumes: nothing
# Effects: gives SQL commands to the database
# Throws: nothing
global inFile, outFile, nextKey
print '%s' % mgi_utils.date()
print 'Initializing'
db.useOneConnection(1)
db.set_sqlLogin(user, password, mgdServer, mgdDB)
results = db.sql('''select max(_Assoc_key) + 1 as nextKey from %s''' % table, 'auto')
nextKey = results[0]['nextKey']
if nextKey == None:
nextKey = 1001
inFilePath = os.environ['INFILE_NAME']
try:
inFile = open(inFilePath, 'r')
except:
exit('Could not open file for reading %s\n' % inFilePath)
outFilePath = '%s/%s.bcp' % (os.environ['OUTPUTDIR'], table)
try:
outFile = open(outFilePath, 'w')
except:
exit('Could not open file for writing %s\n' % outFilePath)
loadLookups()
def init():
global inFile, provider, bcpFilePath, bcpFile
print '%s' % mgi_utils.date()
print 'Initializing'
inFile = sys.stdin
if len(sys.argv) != 2:
print Usage
sys.exit(1)
provider = sys.argv[1]
try:
bcpFilePath = os.environ['BCP_FILE_PATH']
bcpFile = open(bcpFilePath, 'w')
except:
'Could not open file for writing %s\n' % bcpFilePath
sys.exit(1)
if provider == 'ensembl':
loadEnsemblRawBioTypeByGMIDLookup()
elif provider == 'ncbi':
loadNCBIRawBioTypeByGMIDLookup()
else:
'Provider not recognized: %s' % provider
sys.exit(1)
loadSequenceKeyLookup()
loadMarkerTypeKeyLookup()
def init():
# Purpose: initialize file descriptors, create connection to
# the database and load lookups
# Returns: nothing
# Assumes: nothing
# Effects: gives SQL commands to the database
# Throws: nothing
global inFile, outFile, nextKey
print('%s' % mgi_utils.date())
print('Initializing')
db.useOneConnection(1)
db.set_sqlLogin(user, password, mgdServer, mgdDB)
results = db.sql(
'''select max(_Assoc_key) + 1 as nextKey from %s''' % table, 'auto')
nextKey = results[0]['nextKey']
if nextKey == None:
nextKey = 1001
inFilePath = os.environ['INFILE_NAME']
try:
inFile = open(inFilePath, 'r')
except:
exit('Could not open file for reading %s\n' % inFilePath)
outFilePath = '%s/%s.bcp' % (os.environ['OUTPUTDIR'], table)
try:
outFile = open(outFilePath, 'w')
except:
exit('Could not open file for writing %s\n' % outFilePath)
loadLookups()
def header(fp, headerType = "JAX"):
'''
# requires: fp, output file descriptor pointing to an open file
# headerType, string that denotes which header to use
# effects: writes the specified header to the file
'''
jaxheaderfile = '# The Jackson Laboratory - Mouse Genome Informatics (MGI)\n# Copyright 1996, 1999, 2002, 2005, 2008 The Jackson Laboratory\n# All Rights Reserved\n'
#
# always write the JAX header
#
fp.write(jaxheaderfile)
fp.write('# Date Generated: %s\n' % (mgi_utils.date()))
#
# special case
#
if headerType == 'JAX':
fp.write('#\n')
else:
fp.write('# (server = %s, database = %s)\n#\n\n' % (db.get_sqlServer(), db.get_sqlDatabase()))
def run():
# Purpose: iterate through the input file, resolving to keys
# and writing to a bcp file
# Returns: nothing
# Assumes: inFile and outFile are valid file descriptors
# Effects: writes to the file system
# Throws: nothing
global nextKey
print('%s' % mgi_utils.date())
print('Creating bcp file')
for line in inFile.readlines():
(seqId1, qualifier, seqId2) = str.split(line, TAB)
seqId1 = str.strip(seqId1)
qualifier = str.strip(qualifier)
seqId2 = str.strip(seqId2)
if seqId1 in seq1Lookup:
seqKey1 = seq1Lookup[seqId1]
else:
print('SeqId1 %s is not in the database' % seqId1)
continue
if qualifier in qualLookup:
qualKey = qualLookup[qualifier]
else:
print('Qualifier %s is not in the database' % qualifier)
continue
if seqId2 in seq2Lookup:
seqKey2 = seq2Lookup[seqId2]
else:
print('SeqId2 %s is not in the database' % seqId2)
continue
outFile.write('%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' % \
(nextKey, TAB, seqKey1, TAB, qualKey, TAB, seqKey2, \
TAB, CREATEDBY_KEY, TAB, CREATEDBY_KEY, TAB, cdate, TAB, cdate, CRT))
nextKey = nextKey + 1
def exit(status, message = None):
'''
# requires: status, the numeric exit status (integer)
# message (string)
#
# effects:
# Print message to stderr and exits
#
# returns:
#
'''
if message is not None:
sys.stderr.write('\n' + str(message) + '\n')
try:
diagFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
diagFile.close()
accFile.close()
accrefFile.close()
markerFile.close()
except:
pass
sys.exit(status)
def exit(
status, # numeric exit status (integer)
message = None # exit message (string)
):
# Purpose:
# Returns: nothing
# Assumes: nothing
# Effects: nothing
# Throws: nothing
if message is not None:
sys.stderr.write('\n' + str(message) + '\n')
try:
diagFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
diagFile.close()
except:
pass
try:
reportlib.finish_nonps(reportFile)
except:
pass
db.useOneConnection(0)
sys.exit(status)
def init():
global inFile, provider, bcpFilePath, bcpFile
print('%s' % mgi_utils.date())
print('Initializing')
inFile = sys.stdin
if len(sys.argv) != 2:
print(Usage)
sys.exit(1)
provider = sys.argv[1]
try:
bcpFilePath = os.environ['BCP_FILE_PATH']
bcpFile = open(bcpFilePath, 'w')
except:
'Could not open file for writing %s\n' % bcpFilePath
sys.exit(1)
if provider in ('ensembl', 'ensemblreg', 'vistareg'):
loadEnsemblRawBioTypeByGMIDLookup()
elif provider == 'ncbi':
loadNCBIRawBioTypeByGMIDLookup()
else:
print('Provider not recognized: %s' % provider)
sys.exit(1)
loadSequenceKeyLookup()
loadMarkerTypeKeyLookup()
def run():
# Purpose: iterate through the input file, resolving to keys
# and writing to a bcp file
# Returns: nothing
# Assumes: inFile and outFile are valid file descriptors
# Effects: writes to the file system
# Throws: nothing
global nextKey
print '%s' % mgi_utils.date()
print 'Creating bcp file'
for line in inFile.readlines():
(seqId1, qualifier, seqId2) = string.split(line, TAB)
seqId1 = string.strip(seqId1)
qualifier = string.strip(qualifier)
seqId2 = string.strip(seqId2)
if seq1Lookup.has_key(seqId1):
seqKey1 = seq1Lookup[seqId1]
else:
print 'SeqId1 %s is not in the database' % seqId1
continue
if qualLookup.has_key(qualifier):
qualKey = qualLookup[qualifier]
else:
print 'Qualifier %s is not in the database' % qualifier
continue
if seq2Lookup.has_key(seqId2):
seqKey2 = seq2Lookup[seqId2]
else:
print 'SeqId2 %s is not in the database' % seqId2
continue
outFile.write('%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' % \
(nextKey, TAB, seqKey1, TAB, qualKey, TAB, seqKey2, \
TAB, CREATEDBY_KEY, TAB, CREATEDBY_KEY, TAB, cdate, TAB, cdate, CRT))
nextKey = nextKey + 1
def exit(
status, # numeric exit status (integer)
message = None # exit message (string)
):
if message is not None:
sys.stderr.write('\n' + str(message) + '\n')
try:
diagFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
errorFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
diagFile.close()
errorFile.close()
except:
pass
db.useOneConnection(0)
sys.exit(status)
def exit(
status, # numeric exit status (integer)
message=None # exit message (str.
):
if message is not None:
sys.stderr.write('\n' + str(message) + '\n')
try:
diagFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
errorFile.write('\n\nEnd Date/Time: %s\n' % (mgi_utils.date()))
diagFile.close()
errorFile.close()
except:
pass
db.useOneConnection(0)
sys.exit(status)
def deleteByUser():
# Purpose: delete records created by current load
# Returns: nothing
# Assumes: a connection has been made to the database
# Effects: deletes records from a database
# Throws: nothing
print '%s' % mgi_utils.date()
print 'Deleting records for this user'
db.sql('''delete from %s where _CreatedBy_key = %s''' % (table, CREATEDBY_KEY), None)
db.commit()
def createBCP():
outBCP = open('%s/%s.bcp' % (datadir, table), 'w')
print 'sequences1 begin...%s' % (mgi_utils.date())
db.sql('''select s._Object_key as sequenceKey, p._Object_key as probeKey, p._Accession_key
INTO TEMPORARY TABLE sequences1
from ACC_Accession s, ACC_Accession p
where s._MGIType_key = 19
and lower(s.accID) = lower(p.accID)
and p._MGIType_key = 3
and s._LogicalDB_key = p._LogicalDB_key
''', None)
db.sql('create index idx2 on sequences1 (sequenceKey)', None)
db.sql('create index idx3 on sequences1 (probeKey)', None)
db.sql('create index idx4 on sequences1 (_Accession_key)', None)
print 'sequences1 end...%s' % (mgi_utils.date())
print 'deletion begin...%s' % (mgi_utils.date())
db.sql('delete from sequences1 using excluded e where sequences1.probeKey = e._Probe_key', None)
print 'deletion end...%s' % (mgi_utils.date())
db.commit()
print 'sequences2 begin...%s' % (mgi_utils.date())
db.sql('''select s.sequenceKey, s.probeKey, ar._Refs_key as refskey,
ar._ModifiedBy_key as userKey, ar.modification_date as mdate
INTO TEMPORARY TABLE sequences2
from sequences1 s, ACC_AccessionReference ar
where s._Accession_key = ar._Accession_key
''', None)
db.sql('create index idx5 on sequences2 (sequenceKey, probeKey, refsKey, userKey, mdate)', None)
db.sql('create index idx6 on sequences2 (userKey)', None)
db.sql('create index idx7 on sequences2 (mdate)', None)
print 'sequences2 end...%s' % (mgi_utils.date())
print 'final begin...%s' % (mgi_utils.date())
results = db.sql('''select distinct sequenceKey, probeKey, refsKey,
max(userKey) as userKey, max(mdate) as mdate
from sequences2
group by sequenceKey, probeKey, refsKey
''', 'auto')
print 'final end...%s' % (mgi_utils.date())
for r in results:
outBCP.write(mgi_utils.prvalue(r['sequenceKey']) + DL + \
mgi_utils.prvalue(r['probeKey']) + DL + \
mgi_utils.prvalue(r['refsKey']) + DL + \
r['mdate'] + DL + \
mgi_utils.prvalue(r['userKey']) + DL + mgi_utils.prvalue(r['userKey']) + DL + \
loaddate + DL + loaddate + NL)
outBCP.close()
def deleteByUser():
# Purpose: delete records created by current load
# Returns: nothing
# Assumes: a connection has been made to the database
# Effects: deletes records from a database
# Throws: nothing
print('%s' % mgi_utils.date())
print('Deleting records for this user')
db.sql(
'''delete from %s where _CreatedBy_key = %s''' %
(table, CREATEDBY_KEY), None)
db.commit()
def priority2():
# allele names
print 'processing priority 2...%s' % mgi_utils.date()
cmd = '''select distinct a._Allele_key, a.name as label
from ALL_Allele a where a.isWildType = 0
'''
if alleleKey is not None:
cmd = cmd + 'and a._Allele_key = %s\n' % alleleKey
writeRecord(db.sql(cmd, 'auto'), 1, 2, 'AN', 'allele name')
def init():
global diagFile, errorFile, inputFile, errorFileName, diagFileName
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
head, tail = os.path.split(inputFileName)
diagFileName = outputDir + '/' + tail + '.diagnostics'
errorFileName = outputDir + '/' + tail + '.error'
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
return
def init():
global diagFile, errorFile, inputFile, errorFileName, diagFileName
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
head, tail = os.path.split(inputFileName)
diagFileName = outputDir + '/' + tail + '.diagnostics'
errorFileName = outputDir + '/' + tail + '.error'
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
return
def priority3():
# synonyms
print 'processing priority 3...%s' % mgi_utils.date()
cmd = '''select distinct s._Object_key as _Allele_key, s.synonym as label
from MGI_SynonymType st, MGI_Synonym s
where st._MGIType_key = 11
and st._SynonymType_key = s._SynonymType_key
'''
if alleleKey is not None:
cmd = cmd + 'and s._Object_key = %s\n' % alleleKey
writeRecord(db.sql(cmd, 'auto'), 1, 3, 'AY', 'synonym')
def writeRecord(results, labelStatusKey, priority, labelType, labelTypeName):
for r in results:
if labelTypeName is None:
labelTypeName = r['labelTypeName']
outBCP.write(mgi_utils.prvalue(r['_Allele_key']) + COLDL + \
mgi_utils.prvalue(labelStatusKey) + COLDL + \
mgi_utils.prvalue(priority) + COLDL + \
mgi_utils.prvalue(r['label']) + COLDL + \
mgi_utils.prvalue(labelType) + COLDL + \
mgi_utils.prvalue(labelTypeName) + COLDL + \
loaddate + COLDL + \
loaddate + LINEDL)
print 'processed (%d) records...%s' % (len(results), mgi_utils.date())
def init():
global inFile, outFile, seq2Position, qualifier
print '%s' % mgi_utils.date()
print 'Initializing'
inFile = sys.stdin
if len(sys.argv) != 1:
print Usage
sys.exit(1)
outFilePath = os.environ['INFILE_SEQASSOCLOAD']
try:
outFile = open(outFilePath, 'w')
except:
'Could not open file for writing %s\n' % outFilePath
sys.exit(1)
seq2Position = int(os.environ['SEQ_POSITION'])
qualifier = os.environ['QUALIFIER']
def init():
#
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes global file descriptors/file names
#
# returns:
#
global diagFile, inputerrorFile, ens1errorFile, ens2errorFile, mgierrorFile
global mgiens
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
diagFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
db.useOneConnection(1)
def createBCP():
print 'Creating %s.bcp...%s' % (table, mgi_utils.date())
outBCP = open('%s/%s.bcp' % (datadir, table), 'w')
cmd = 'select distinct mc._Map_key, mc.version, t2.abbreviation as mapUnits, ' + \
'mcf._Object_key, mcf.startCoordinate, mcf.endCoordinate, mcf.strand, ' + \
'c.chromosome, t3.term as provider ' + \
'from MAP_Coordinate mc, MAP_Coord_Feature mcf, ' + \
'MRK_Chromosome c, SEQ_Sequence s, VOC_Term t1, VOC_Term t2, VOC_Term t3 ' + \
'where mc._MapType_key = t1._Term_key ' + \
'and t1.term = \'Assembly\' ' + \
'and mc._Units_key = t2._Term_key ' + \
'and mc._MGIType_key = 27 ' + \
'and mc._Object_key = c._Chromosome_key ' + \
'and mc._Map_key = mcf._Map_key ' + \
'and mcf._MGIType_key = 19 ' + \
'and mcf._Object_key = s._Sequence_key ' + \
'and s._SequenceProvider_key = t3._Term_key'
results = db.sql(cmd, 'auto')
for r in results:
outBCP.write(str(r['_Map_key']) + DL + \
str(r['_Object_key']) + DL + \
r['chromosome'] + DL + \
str(r['startCoordinate']) + DL + \
str(r['endCoordinate']) + DL + \
str(r['strand']) + DL + \
str(r['mapUnits']) + DL + \
str(r['provider']) + DL + \
str(r['version']) + DL + \
str(userKey) + DL + str(userKey) + DL + \
loaddate + DL + loaddate + NL)
outBCP.close()
# bcp file path
bcpFilePath = os.environ['SGM_BCPFILE']
# input file path
inputFilePath = os.environ['SGM_BIOTYPEFILE']
# Logical DB for this load
ldbName = os.environ['SGM_LOGICALDB']
# file descriptors
inFile = ''
bcpFile = ''
# timestamp for creation/modification date
cdate = mgi_utils.date('%m/%d/%Y') # current date
#
# lookups
#
# loaded from provider input file - maps gene model ID to biotype
bioTypeByGMIDLookup = {} # {GM Id:BioType, ...}
# loaded from mcv vocabulary
featureTypeLookup = []
# loaded from db by provider - maps a GM Id to its _Sequence_key
seqKeyByGMIDLookup = {} # {GM Id:_Sequence_key, ...}
# Provider we are loading MGP or MGI B6 Strain Gene Model
provider = ldbName
# Effects: None
# Throws: Nothing
fpOrthoFile.close()
fpExprFile.close()
fpLoadFile.close()
fpQcRpt.close()
# close the database connection
db.useOneConnection(0)
return
###--- main program ---###
print '%s' % mgi_utils.date()
print 'initializing'
init()
print 'processing input files'
processInputFiles()
print 'processing clusters'
process()
print 'writing reports'
writeReports()
print 'closing files'
closeFiles()
def init():
global diagFile, errorFile, inputFile, errorFileName, diagFileName
global strainFile, markerFile, accFile, annotFile
global noteFile
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
fdate = mgi_utils.date('%m%d%Y') # current date
head, tail = os.path.split(inputFileName)
diagFileName = tail + '.' + fdate + '.diagnostics'
errorFileName = tail + '.' + fdate + '.error'
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
inputFile = open(inputFileName, 'r', encoding="latin-1")
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
strainFile = open(strainFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % strainFileName)
try:
markerFile = open(markerFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % markerFileName)
try:
accFile = open(accFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % accFileName)
try:
noteFile = open(noteFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % noteFileName)
try:
annotFile = open(annotFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % annotFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
return
def initialize():
global diagFile, errorFile, inputFile, errorFileName, diagFileName
global alleleFile, mutationFile, mutantFile, refFile
global accFile, accRefFile, noteFile, noteChunkFile, annotFile
global newAlleleFile
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
head, tail = os.path.split(inputFileName)
diagFileName = outputDir + '/' + tail + '.diagnostics'
errorFileName = outputDir + '/' + tail + '.error'
newAlleleFileName = outputDir + '/' + tail + '.new'
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
newAlleleFile = open(newAlleleFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % newAlleleFileName)
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
alleleFile = open(alleleFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % alleleFileName)
try:
mutationFile = open(mutationFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % mutationFileName)
try:
mutantFile = open(mutantFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % mutantFileName)
try:
refFile = open(refFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % refFileName)
try:
accFile = open(accFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % accFileName)
try:
accRefFile = open(accRefFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % accRefFileName)
try:
noteFile = open(noteFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % noteFileName)
try:
noteChunkFile = open(noteChunkFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % noteChunkFileName)
try:
annotFile = open(annotFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % annotFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
fp.write(r['goaccID'] + TAB)
fp.write(r['pValue'] + TAB)
fp.write(r['dbuser'] + CRT)
badCount+=1
return badCount
#
# Main
#
#Note: Harold said for now the generated report should be stored under the tr directory
# When he changes his mind, we will set outputdir=os.environ['QCOUTPUTDIR']
#
fp = reportlib.init(sys.argv[0], outputdir = os.environ['QCOUTPUTDIR'])
fp.write("Date: %s\n\n" % (mgi_utils.date()))
#
# Get all GO/Marker evidence annotations records for J:73065 -> _refs_key=74017
# _annottyp_key=1000 (GO/Marker); exclude annotations where _user_key=1503 (login=GOC)
# or user login like 'NOCTUA%'
db.sql('''
select ve._annotevidence_key,va._term_key,va._object_key,m.login
into temporary table goevidence
from voc_evidence ve,voc_annot va, mgi_user m
where ve._refs_key=74017
and ve._createdby_key= m._user_key
and m._user_key not in (1503)
and ve._annot_key=va._annot_key
and va._annottype_key=1000
and not exists (select 1 from mgi_user m where ve._createdby_key = m._user_key and m._user_key in (1503))
def init():
'''
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes local DBMS parameters
# 3. Initializes global file descriptors/file names
# 4. Initializes global keys
#
# returns:
#
'''
global inputFile, diagFile, errorFile, errorFileName, diagFileName
global passwordFileName, noteFileName
global exptFile, exptMarkerFile, accFile, noteFile
global inputFileName, exptFileName, exptMarkerFileName, accFileName
global mode, exptType
try:
optlist, args = getopt.getopt(sys.argv[1:], 'S:D:U:P:M:I:R:E:C:')
except:
showUsage()
#
# Set server, database, user, passwords depending on options
# specified by user.
#
server = ''
database = ''
user = ''
password = ''
inputFileName = ''
jnum = ''
createdBy = ''
for opt in optlist:
if opt[0] == '-S':
server = opt[1]
elif opt[0] == '-D':
database = opt[1]
elif opt[0] == '-U':
user = opt[1]
elif opt[0] == '-P':
passwordFileName = opt[1]
elif opt[0] == '-M':
mode = opt[1]
elif opt[0] == '-I':
inputFileName = opt[1]
elif opt[0] == '-E':
exptType = re.sub('"', '', opt[1])
else:
showUsage()
# User must specify Server, Database, User and Password
password = string.strip(open(passwordFileName, 'r').readline())
if server == '' or \
database == '' or \
user == '' or \
password == '' or \
mode == '' or \
inputFileName == '' or \
exptType == '':
showUsage()
# Initialize db.py DBMS parameters
db.set_sqlLogin(user, password, server, database)
db.useOneConnection(1)
diagFileName = 'mappingload.diag'
errorFileName = 'mappingload.error'
exptFileName = 'MLD_Expts.mapping.bcp'
exptMarkerFileName = 'MLD_Expt_Marker.mapping.bcp'
accFileName = 'ACC_Accession.mapping.bcp'
noteFileName = 'MLD_Notes.mapping.bcp'
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
exptFile = open(exptFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % exptFileName)
try:
exptMarkerFile = open(exptMarkerFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % exptMarkerFileName)
try:
accFile = open(accFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % accFileName)
try:
noteFile = open(noteFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % noteFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
# Set Log File Descriptor
db.set_commandLogFile(diagFileName)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (server))
diagFile.write('Database: %s\n' % (database))
diagFile.write('User: %s\n' % (user))
diagFile.write('Input File: %s\n' % (inputFileName))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
#
'''
import sys
import os
import string
import mgi_utils
import reportlib
import db
import time
db.setTrace()
CRT = reportlib.CRT
TAB = reportlib.TAB
date = mgi_utils.date('%m/%d/%Y %H:%M:%S') #("%Y-%m-%d")
mAssembly = os.getenv('MOUSE_ASSEMBLY')
# ENSEMBL GFF Configuration
# zipped version of ensembl gff - filename from source
sourceEnsemblGFF = os.getenv('ENSEMBL_GFF')
# local unzipped sourceEnsemblGFF file from which to parse bounds
ensemblGFF = os.getenv('ENSEMBL_GFF_DEFAULT')
# just the file name w/o extension
baseEnsemblGFF = os.path.splitext(os.path.basename(sourceEnsemblGFF))[0]
# URL from which we get sourceEnsembGFF
#fpStudentRpt.close()
db.useOneConnection(0)
return 0
# end closefiles ()--------------------------------------------
#
# Main
#
# -------------------------------------------------------------
START_TIME = time.time()
print 'Start time: %s' % mgi_utils.date()
sys.stdout.flush()
init()
elapsed_time = time.time() - START_TIME
print 'TIME to run init function %s' % time.strftime("%H:%M:%S",
time.gmtime(elapsed_time))
sys.stdout.flush()
# -------------------------------------------------------------
TIME = time.time()
process()
elapsed_time = time.time() - TIME
print 'TIME to run process function %s' % time.strftime(
"%H:%M:%S", time.gmtime(elapsed_time))
sys.stdout.flush()
def init():
'''
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes local DBMS parameters
# 3. Initializes global file descriptors/file names
# 4. Initializes global keys
#
# returns:
#
'''
global inputFile, diagFile, errorFile, errorFileName, diagFileName
global transTypeFile, transFile
global transTypeFileName, transFileName
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
# the default output file names are bases on 'inputFileName'
head, fileName = os.path.split(inputFileName)
# rename 'head'
head = outputFileDir
fdate = mgi_utils.date('%m%d%Y') # current date
diagFileName = head + '/' + fileName + '.' + fdate + '.diagnostics'
print diagFileName
errorFileName = head + '/' + fileName + '.' + fdate + '.error'
print errorFileName
transTypeFileName = head + '/' + fileName + '.' + fdate + '.MGI_TranslationType.bcp'
print transTypeFileName
transFileName = head + '/' + fileName + '.' + fdate + '.MGI_Translation.bcp'
print transFileName
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
transTypeFile = open(transTypeFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % transTypeFileName)
try:
transFile = open(transFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % transFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
# Set Log File Descriptor
db.set_sqlLogFD(diagFile)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
diagFile.write('Input File: %s\n' % (inputFileName))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
fpDiag.close()
fpStudentRpt.close()
db.useOneConnection(0)
return 0
# end closefiles ()--------------------------------------------
#
# Main
#
# -------------------------------------------------------------
START_TIME = time.time()
print('Start time: %s' % mgi_utils.date())
sys.stdout.flush()
init()
elapsed_time = time.time() - START_TIME
print('TIME to run init function %s' % time.strftime("%H:%M:%S", time.gmtime(elapsed_time)))
sys.stdout.flush()
# -------------------------------------------------------------
TIME = time.time()
process()
elapsed_time = time.time() - TIME
print('TIME to run process function %s' % time.strftime("%H:%M:%S", time.gmtime(elapsed_time)))
sys.stdout.flush()
# -------------------------------------------------------------
DB_PREFIX = 'MGI'
SPECIES = 'taxon:10090'
DBTYPE_MARKER = 'gene'
DBTYPE_ISOFORM = 'protein'
DBTYPE_RNA = 'transcript'
TAB = reportlib.TAB
CRT = reportlib.CRT
db.useOneConnection(1)
fp = reportlib.init('mgi', fileExt = '.gpi', outputdir = os.environ['REPORTOUTPUTDIR'], printHeading = None)
fp.write('!gpi-version: 1.2\n')
fp.write('!date: %s $\n' % (mgi_utils.date("%m/%d/%Y")))
fp.write('!\n')
fp.write('! from Mouse Genome Database (MGD) & Gene Expression Database (GXD)\n')
fp.write('!\n')
fp.write('! DB required 1 1 MGI\n')
fp.write('! DB_Object_ID required 1 2/17 MGI:87870\n')
fp.write('! DB_Object_Symbol required 1 3 Acat1\n')
fp.write('! DB_Object_Name optional 0 or greater 10 acetyl-Coenzyme A acetyltransferase 1\n')
fp.write('! DB_Object_Synonym(s) optional 0 or greater 11 Acat|6330585C21Rik\n')
fp.write('! DB_Object_Type required 1 12 gene\n')
fp.write('! Taxon required 1 13 taxon:10090\n')
fp.write('! Parent_Object_ID optional 0 or 1 - if DB_Object_ID = Isoform, then MGI:id of the Isoform\n')
fp.write('! DB_Xref(s) optional 0 or greater - if DB_Object_ID = Isoform, then UniProtKB:id of the Isoform\n')
fp.write('! Properties optional 0 or greater - blank\n\n')
fp.write('! DB_Object_Type = "gene", DB = "MGI", DB_Object_ID = "MGI:xxxx"\n')
fp.write('! DB_Object_Type = "protein", DB = "PR", DB_Object_ID = "xxxx", Parent_Object_ID = "MGI:MGI:xxxx", DB_Xref = "UniProtDB:xxx"\n')
import accessionlib
import db
#globals
logicalDBDict = {} # logical DB
mgiTypeDict = {} # mgi type
markerDict = {} # markers
objectDict = {} # objects
probeDict = {} # probes
referenceDict = {} # references
termDict = {} # terms
userDict = {} # users
markerTypeDict = {} # marker types
loaddate = mgi_utils.date('%m/%d/%Y %H:%M:%S') # current date
# Purpose: verifies the Logical DB value
# Returns: 0 if the Logical DB value does not exist in MGI
# else the primary key of the Logical DB
# Assumes: nothing
# Effects: initializes the Logical DB dictionary for quicker lookup
# Throws: nothing
def verifyLogicalDB(
logicalDB, # the Logical DB value from the input file (str.
lineNum, # the line number (from the input file) on which this value was found (integer)
errorFile # error file
):
def init():
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes local DBMS parameters
# 3. Initializes global file descriptors/file names
# 4. Initializes global keys
#
# returns:
#
global diagFileName, errorFileName, synFileName
global inputFile, diagFile, errorFile, synFile
global mgiTypeKey, createdByKey, referenceKey
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
head, tail = os.path.split(inputFileName)
diagFileName = logDir + '/' + tail + '.diagnostics'
errorFileName = logDir + '/' + tail + '.error'
synFileName = 'MGI_Synonym.bcp'
print inputFileName
print logDir
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
synFile = open(outputDir + '/' + synFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % synFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
diagFile.write('Object Type: %s\n' % (mgiType))
diagFile.write('Input File: %s\n' % (inputFileName))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
mgiTypeKey = loadlib.verifyMGIType(mgiType, 0, errorFile)
createdByKey = loadlib.verifyUser(createdBy, 0, errorFile)
# if reference is J:0, then no reference is given
if jnum == 'J:0':
referenceKey = ''
else:
referenceKey = loadlib.verifyReference(jnum, 0, errorFile)
# exit if we can't resolve mgiType, createdBy or jnum
if mgiTypeKey == 0 or \
createdByKey == 0 or \
referenceKey == 0:
exit(1)
if mode == 'reload':
print 'mode is: %s, deleting synonyms' % mode
sys.stdout.flush()
db.sql('delete from MGI_Synonym ' + \
'where _MGIType_key = %d ' % (mgiTypeKey) + \
'and _CreatedBy_key = %d ' % (createdByKey), None)
def initialize():
# Purpose: open file descriptors; write timestamps to log files
# Returns: 1 if error, else 0
# Assumes: Nothing
# Effects: Sets global variables, exits if a file can't be opened,
# creates files in the file system
global fpDiagFile, fpErrorFile, fpInputFile, errorFileName, diagFileName
global fpAlleleFile, fpMutationFile, fpRefFile
global fpAccFile, fpNoteFile, fpAnnotFile
global fpNewAlleleRptFile
db.useOneConnection(1)
#db.set_sqlUser(user)
#db.set_sqlPasswordFromFile(passwordFileName)
head, tail = os.path.split(inputFileName)
diagFileName = outputDir + '/' + tail + '.diagnostics'
errorFileName = outputDir + '/' + tail + '.error'
try:
fpDiagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
fpErrorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
fpNewAlleleRptFile = open(newAlleleRptFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % newAlleleRptFileName)
try:
fpInputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
fpAlleleFile = open(alleleFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % alleleFileName)
try:
fpMutationFile = open(mutationFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % mutationFileName)
try:
fpRefFile = open(refFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % refFileName)
try:
fpAccFile = open(accFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % accFileName)
try:
fpNoteFile = open(noteFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % noteFileName)
try:
fpAnnotFile = open(annotFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % annotFileName)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
fpDiagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
fpDiagFile.write('Server: %s\n' % (db.get_sqlServer()))
fpDiagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
fpErrorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
return 0
def init():
'''
# requires:
#
# effects:
# 1. Processes command line options
# 2. Initializes local DBMS parameters
# 3. Initializes global file descriptors/file names
#
# returns:
#
'''
global inputFile, diagFile, errorFile, errorFileName, diagFileName
global passwordFileName
global noteFile, noteFileName, noteChunkFile, noteChunkFileName, sqlFile, sqlFileName
global mode
global noteTypeName
global objectTypeKey, createdByKey
global mgiObjects
try:
optlist, args = getopt.getopt(sys.argv[1:], 'S:D:U:P:M:I:O:T:')
except:
showUsage()
#
# Set server, database, user, passwords depending on options
# specified by user.
#
server = None
database = None
user = None
password = None
for opt in optlist:
if opt[0] == '-S':
server = opt[1]
elif opt[0] == '-D':
database = opt[1]
elif opt[0] == '-U':
user = opt[1]
elif opt[0] == '-P':
passwordFileName = opt[1]
elif opt[0] == '-M':
mode = opt[1]
elif opt[0] == '-I':
inputFileName = opt[1]
elif opt[0] == '-O':
objectType = opt[1]
elif opt[0] == '-T':
noteTypeName = re.sub('"', '', opt[1])
else:
showUsage()
# Initialize db.py DBMS parameters
password = string.strip(open(passwordFileName, 'r').readline())
db.set_sqlLogin(user, password, server, database)
db.useOneConnection(1)
head, tail = os.path.split(inputFileName)
diagFileName = tail + '.diagnostics'
errorFileName = tail + '.error'
noteFileName = tail + '.' + noteTable + '.bcp'
noteChunkFileName = tail + '.' + noteChunkTable + '.bcp'
sqlFileName = tail + '.sql'
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
noteFile = open(noteFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % noteFileName)
try:
noteChunkFile = open(noteChunkFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % noteChunkFileName)
try:
sqlFile = open(sqlFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % sqlFileName)
# Set Log File Descriptor
try:
db.set_sqlLogFD(diagFile)
except:
pass
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (server))
diagFile.write('Database: %s\n' % (database))
diagFile.write('User: %s\n' % (user))
diagFile.write('Input File: %s\n' % (inputFileName))
diagFile.write('Object Type: %s\n' % (objectType))
diagFile.write('Note Type: %s\n' % (noteTypeName))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
objectTypeKey = accessionlib.get_MGIType_key(objectType)
createdByKey = loadlib.verifyUser(db.get_sqlUser(), 0, errorFile)
results = db.sql('''
select accID, _Object_key from ACC_Accession
where _MGIType_key = %s
and _LogicalDB_key = 1
and prefixPart = 'MGI:'
and preferred = 1
''' % (objectTypeKey), 'auto')
for r in results:
mgiObjects[r['accID']] = r['_Object_key']
# 03/11/2014 sc Initial development
#
###########################################################################
import sys
import os
import string
import db
import mgi_utils
#
# CONSTANTS
#
TAB = '\t'
CRT = '\n'
DATE = mgi_utils.date("%m/%d/%Y")
USAGE='fearload.py'
#
# GLOBALS
#
# input file
inFile = os.environ['INPUT_FILE_DEFAULT']
# output bcp files
relationshipFile = os.environ['RELATIONSHIP_BCP']
propertyFile = os.environ['PROPERTY_BCP']
noteFile = os.environ['NOTE_BCP']
# file descriptors
def init():
global diagFile, errorFile, inputFile, errorFileName, diagFileName
global outSetFile, outMemberFile
global setKey, setMemberKey, createdByKey, mgiTypeKey, useSetKey
global DEBUG
db.useOneConnection(1)
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFileName)
diagFileName = '%s/setload.diagnostics' % (outputDir)
errorFileName = '%s/setload.error' % (outputDir)
try:
diagFile = open(diagFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % diagFileName)
try:
errorFile = open(errorFileName, 'w')
except:
exit(1, 'Could not open file %s\n' % errorFileName)
try:
inputFile = open(inputFileName, 'r')
except:
exit(1, 'Could not open file %s\n' % inputFileName)
# Output Files
try:
fullPathSetFile = '%s/%s' % (outputDir, outSetFileName)
outSetFile = open(fullPathSetFile, 'w')
except:
exit(1, 'Could not open file %s\n' % fullPathSetFile)
try:
fullPathMemberFile = '%s/%s' % (outputDir, outMemberFileName)
outMemberFile = open(fullPathMemberFile, 'w')
except:
exit(1, 'Could not open file %s\n' % fullPathMemberFile)
# Log all SQL
db.set_sqlLogFunction(db.sqlLogAll)
diagFile.write('Start Date/Time: %s\n' % (mgi_utils.date()))
diagFile.write('Server: %s\n' % (db.get_sqlServer()))
diagFile.write('Database: %s\n' % (db.get_sqlDatabase()))
errorFile.write('Start Date/Time: %s\n\n' % (mgi_utils.date()))
if mode == 'preview':
DEBUG = 1
bcpon = 0
elif mode != 'load':
exit(1, 'Invalid Processing Mode: %s\n' % (mode))
results = db.sql('select max(_Set_key) + 1 as maxKey from MGI_Set', 'auto')
setKey = results[0]['maxKey']
createdByKey = loadlib.verifyUser(createdBy, 0, errorFile)
mgiTypeKey = loadlib.verifyMGIType(setType, 0, errorFile)
#
# use existing MGI_Set, or create a new one
#
results = db.sql(
'select _Set_key from MGI_Set where _MGIType_key = %s and name = \'%s\''
% (mgiTypeKey, setName), 'auto')
if len(results) > 0:
for r in results:
setKey = r['_Set_key']
# delete/reload
db.sql('delete from MGI_SetMember where _Set_key = %s' % (setKey),
None)
else:
outSetFile.write(str(setKey) + TAB + \
str(mgiTypeKey) + TAB + \
str(setName) + TAB + \
'1' + TAB + \
str(createdByKey) + TAB + str(createdByKey) + TAB + \
loaddate + TAB + loaddate + CRT)
results = db.sql(
'select max(_SetMember_key) + 1 as maxKey from MGI_SetMember', 'auto')
setMemberKey = results[0]['maxKey']
return
# Effects: None
# Throws: Nothing
fpOrthoFile.close()
fpExprFile.close()
fpLoadFile.close()
fpQcRpt.close()
# close the database connection
db.useOneConnection(0)
return
###--- main program ---###
print('%s' % mgi_utils.date())
print('initializing')
init()
print('processing input files')
processInputFiles()
print('processing clusters')
process()
print('writing reports')
writeReports()
print('closing files')
closeFiles()
COLDL = '\t'
LINEDL = '\n'
NOTE_BCP_FILE = OUT_DIR + "/MGI_Note.bcp"
NOTECHUNK_BCP_FILE = OUT_DIR + "/MGI_NoteChunk.bcp"
# note type for annotation extension display/link
DISPLAY_NOTE_TYPE_KEY = 1045
# MGI Type key for voc_evidence_property
PROPERTY_MGITYPE_KEY = 41
VOCAB_TERM_MGITYPE_KEY = 13
MARKER_MGITYPE_KEY = 2
# current date
CDATE = mgi_utils.date("%m/%d/%Y")
# use mgd_dbo
CREATEDBY_KEY = 1001
# temp table for joining against ACC_Accession
TEMP_ID_TABLE = "tmp_annot_id"
# External database links
# (Logical DB name, Actual DB name)
DATABASE_PROVIDERS = [
# Cell Ontology
('CL','Cell Ontology'),
# Ensembl
('Ensembl Gene Model','Ensembl Gene Model'),
# PR: IDs
# GLOBALS
#
user = os.environ['MGD_DBUSER']
passwordFile = os.environ['MGD_DBPASSWORDFILE']
inputFile = os.environ['GENSATLOAD_INPUTFILE']
rptFile = os.environ['GENSATLOAD_RPTFILE']
accBCPFile = os.environ['GENSATLOAD_ACC_BCPFILE']
tempTable = os.environ['GENSAT_TEMP_TABLE']
gensatLogicalDB = os.environ['GENSAT_LOGICAL_DB']
egLogicalDB = os.environ['EG_LOGICAL_DB']
markerMGIType = os.environ['MARKER_MGITYPE']
createdBy = os.environ['GENSAT_CREATED_BY']
loadDate = loadlib.loaddate
timestamp = mgi_utils.date()
#
# Purpose: Perform initialization steps.
# Returns: Nothing
# Assumes: Nothing
# Effects: Sets global variables.
# Throws: Nothing
#
def init():
global accKey, gensatLogicalDBKey, egLogicalDBKey
global markerMGITypeKey, createdByKey
db.set_sqlUser(user)
db.set_sqlPasswordFromFile(passwordFile)
import loadlib
### Constants ###
# Biotype vocab keys
ENSEMBL_VOCAB_KEY = 103
NCBI_VOCAB_KEY = 104
MGP_VOCAB_KEY = 136
ENSEMBLREG_VOCAB_KEY = 176
VISTAREG_VOCAB_KEY = 175
# MCV vocab key
MCV_VOCAB_KEY = 79
createdByKey = 1001
cdate = mgi_utils.date('%m/%d/%Y') # current date
# sanity check error messages
INVALID_VOCAB_ERROR = "Invalid BioType Vocab (row %d): %s"
INVALID_BIOTYPE_TERM_ERROR = "Invalid BioType Term (row %d): %s, for vocab %s"
INVALID_MARKER_TYPE_ERROR = "Invalid Marker Type Term (row %d): %s"
INVALID_MCV_TERM_ERROR = "Invalid MCV/Feature Type Term (row %d): %s"
INVALID_PRIMARY_FEATURE_ERROR = "Invalid Primary Feature Type Term (row %d): %s"
### Globals ###
DEBUG = 0 # set DEBUG to false unless preview mode is selected
bcpon = 1 # can the bcp files be bcp-ed into the database? default is yes (1).
inputFile = '' # file descriptor
outputFile = '' # file descriptor