def __init__(self, obid=None):
op.__init__(self)
if obid != None:
con = databaseModule.getConnection()
self.initFromDatabase(obid, con)
con.close()
def __init__(self, obid=None):
ob.__init__(self)
self.MAX_INLINE_SEQUENCE_LENGTH = 50000
self.MAX_INLINE_FEATURE_COUNT = 100
self.AboutMe['default'][0]['heading'] = 'BRDF Sequence Object'
self.AboutMe['default'][0]['text'] = """
This page displays a sequence object from the BRDF database. Contents include:
<ul>
<li> Basic sequence information - the sequence, length, molecule type
<li> Blast or other search annotation if available
<li> A graphical feature display if features have been loaded
<li> A trace viewer if there is a trace file linked to the sequence. The clipped off ends
are usually shaded gray in this view, the sequence corresponding to that shown in the basic
sequence information panel is unshaded. In some cases there may be no shading, if the
drawing code could not determine where the clipping started. Some ABI files seem to
adopt a convention of starting the clipped sequence at position zero, and it may be possible to
identify the start of clipping using this rule, even when we have not shaded the clipped
region.
</ul>
In the BRDF database schema, sequence objects are stored in a table called
biosequenceob. Various types of sequence can be stored in this table - for example,
nucleotide, protein, mRNA sequence, genomic sequence. Also, various types of
sequence model can be stored in addition to the common models of nucleic and amino acid symbol strings -
for example, other models of sequences include regular expressions and HMMs.
</p>
The type of a sequence object is stored in the sequencetype field of this table.
The current set of types that are supported in this database are recorded in an ontology called
AGSEQUENCE_ONTOLOGY (you can browse this ontology by selecting ONTOLOGIES from the drop down list
of search types on the main page, and entering AGSEQUENCE_ONTOLOGY as the search phrase).
<p/>
A Sequence Object in the BRDF database may have relationships with other types of object
in the database. For example, if the sequence is a primary sequence resulting from lab work, then
there is a relation between the biological sample used in the sequencing experiment and one or more
lab resources (vectors, primers); a sequence may be the product of a gene and thus related to
a BRDF geneticob record; if we have searched sequence databases (e.g. via BLAST) and
found similar sequences and stored these search results in the BRDF schema, then this involves a
relationship with a database search run and another (hit) biosequence. All of the possible relationships that a
sequence object may have with other objects in the database are depicted by icons in the information
map section of the page.
<p/>
The icon is composed of the sequenceob symbol connected to the symbols for the related objects
by horizontal lines. Where the database does not yet contain any related objects for a sequence,
the icon for that relationship is greyed out.
<p/>
We may store various types of facts about a given sequence object - for example, features of the
sequence; or miscellaneous facts specific to a given project. Each type of fact supported by the
BRDF is depicted by a line connecting the sequence symbol, to a square box labelled info, with the type of
fact appearing to the right of the icon. Where the database does not yet contain any facts
of a given type for a sequence object, the info icon is greyed out.
"""
if obid != None:
con = databaseModule.getConnection()
self.initFromDatabase(obid, con)
con.close()
def processExtracts(args):
print "in processExtracts"
config = configs[args["format"]]
connection = databaseModule.getConnection()
extractcur = connection.cursor()
for itemkey in config:
if re.search("^extract", itemkey) != None:
sql = config[itemkey] % args
print "executing %s" % sql
if args["action"] == "test_extract":
print "(but only testing)"
elif args["action"] == "do_extract":
extractcur.execute(sql)
myrow = extractcur.fetchone()
outfile = "%s.extract" % myrow[0]
print "writing %s" % outfile
mywriter = file(outfile, "w")
colnames = [item[0] for item in extractcur.description]
mywriter.write(reduce(lambda x, y: x + '\t' + y, colnames[1:]))
mywriter.write("\n")
while myrow != None:
if outfile != "%s.extract" % myrow[0]:
mywriter.close()
outfile = "%s.extract" % myrow[0]
mywriter = file(outfile, "w")
mywriter.write(
reduce(lambda x, y: x + '\t' + y, colnames[1:]))
mywriter.write("\n")
print "writing %s" % outfile
mywriter.write(
reduce(lambda x, y: str(x) + '\t' + str(y),
[{
True: '',
False: item
}[item == None] for item in myrow[1:]]))
mywriter.write("\n")
myrow = extractcur.fetchone()
mywriter.close()
extractcur.close()
connection.close()
def processUpdates(args):
print "in processUpdates"
config = configs[args["format"]]
connection = databaseModule.getConnection()
updatecur = connection.cursor()
for itemkey in config:
if re.search("^update", itemkey) != None:
sql = config[itemkey] % args
print "executing %s" % sql
if args["action"] == "test_update":
print "(but only testing)"
elif args["action"] == "do_update":
updatecur.execute(sql)
print "rowcount = %s" % updatecur.rowcount
updatecur.close()
connection.close()
def processFiles(args):
print "in processFiles processing %s" % str(args)
# create data source object and make it executable
if "datasourcelsid" not in args or "action" not in args or "format" not in args:
print "Useage : nutrigenomicsForms logpath=mypath datasourcelsid=mylsid"
connection = databaseModule.getConnection()
archive = dataSourceOb()
archive.initFromDatabase(args["datasourcelsid"], connection)
print "archive initialised , with fields %s" % str(archive.databaseFields)
# theoretically the datasourcetype should allow us to work out how to list contents but
# users may have specified some random type, so work it out from the name
if re.search("\.zip$", args["datasourcelsid"]) != None:
print "(looks like a zip archive)"
else:
print "Sorry , only zip archives currently supported"
sys.exit(1)
print "getting archive contents using unzip -l and parsing the output..."
archive.databaseFields["datasourcetype"] = "Executable"
#archive.databaseFields["datasourcecontent"] = 'unzip -l "%(physicalsourceuri)s"'%archive.databaseFields
archive.databaseFields[
"datasourcecontent"] = 'unzip -c "%(physicalsourceuri)s" | grep inflating' % archive.databaseFields
(errorcode, output) = archive.execute(connection, outfile=None)
print "execute returned %s" % errorcode
if errorcode != 0:
print "exiting as execute returned error code"
sys.exit(errorcode)
print "archive contains : \n%s" % output
# example :
# inflating: Normalized and raw data, reference as CH2, for slide IL-10week7sa75mediumfinal
# inflating: Normalized and raw data, reference as CH2, for slide IL-10week7sa77mediumfinal
# inflating: Normalized and raw data, reference as CH2, for slide IL-10week7sa78mediumfinal
# get an array of files
records = re.split("\r*\n\r*", output)
files = [re.split("inflating: ", record)[1].strip() for record in records]
print "processing files : %s" % files
insertCursor = connection.cursor()
# for each file in the archive
for geofile in files:
# execute the datasource to extract the file contents
print "processing %s" % geofile
archive.databaseFields[
"datasourcecontent"] = 'unzip -p "%(physicalsourceuri)s" "%(geofile)s"' % {
"physicalsourceuri":
archive.databaseFields["physicalsourceuri"],
"geofile": geofile
}
(errorcode, output) = archive.execute(connection, outfile=None)
print "execute returned %s" % errorcode
#if args["action"] == "test":
# print "file content : \n%s"%output
# parse contents into records
records = [
item.strip() for item in re.split("\r*\n\r*", output)
if len(item.strip()) > 1
]
records = [
re.split("\t", item) for item in records
if len(re.split("\t", item)) > 1
]
print records[1:5]
print ".\n.\n.\netc"
# check the format of the contents : column headngs
format = configs[args["format"]]
headings = [item.upper() for item in records[0]]
headings = [re.sub('^"', '', item.upper()) for item in headings]
headings = [re.sub('"$', '', item.upper()) for item in headings]
print "verifying format %s" % format
for field in headings:
if field not in format["columns"]:
print "error - unsupported column name %s (columns are : %s)" % (
field, str(format["columns"]))
sys.exit(1)
print "data format looks OK"
insertCount = 0
filerecnum = 0
for record in records[1:]:
filerecnum += 1
# translate "null" , "NULL" , "NA" etc
record = [re.sub('^"', '', item) for item in record]
record = [re.sub('"$', '', item) for item in record]
record = [{
True: None,
False: item
}[item.lower() in ("null", "na", '"null"', '"na"')]
for item in record]
fieldDict = dict(zip(headings, record))
fieldDict["geofile"] = geofile
fieldDict["filerecnum"] = filerecnum
if args["action"] == "test_import":
print("will update database with \n%s \n(but only testing)" %
format["insert_sql"] % fieldDict)
insertCount += 1
if insertCount % 10 == 9:
break
else:
insertCursor.execute(format["insert_sql"], fieldDict)
insertCount += 1
if insertCount % 50 == 49:
print "inserted %s" % insertCount
connection.commit()
if args["action"] == "do_import":
connection.commit()
# set up as a new type in contributed table
print "installing data as contributed type %(datasetname)s..." % args
sql = """
select installContributedTable('GEOSubmissionData','%(datasetname)s','%(datasetname)s')
""" % args
insertCursor.execute(sql)
connection.commit()
connection.close()
#Script to import the user-friendly field-names from the csv file into the database.
#Import the postgres module
import csv
import databaseModule
connection = databaseModule.getConnection()
queryCursor = connection.cursor()
sql = "select max(obid) from ontologyOb"
queryCursor.execute(sql)
print 'Max obid for ontologyOb is: ' + str(queryCursor.fetchall()[0][0])
sql = "select max(obid) from ontologyTermFact"
queryCursor.execute(sql)
print 'Max obid for ontologyTermFact is: ' + str(queryCursor.fetchall()[0][0])
reader = csv.reader(open("DatabaseNames.csv"))
for table_name, column_name, user_friendly_name in reader:
#Rename the "friendly" name when not displayed to NULL...
if user_friendly_name == '** not displayed **' :
user_friendly_name = 'NULL'
else : #...otherwise, put single-quotes around the name, to avoid the NULL being entered as a string
user_friendly_name = "'" + user_friendly_name + "'"
#If the table doesn't already have an entry in the ontologyob table, create one
sql = """ select obid from ontologyOb where ontologyName = 'BRDFTABLE_%s';""" % table_name
queryCursor.execute(sql)
if len(queryCursor.fetchall()) == 0 :
sql = """
def myHTMLSummary(self, table, width="90%",context='default'):
summaryFieldNames = ['sequencename','sequencetype','sequencedescription','seqlength','seqcomment','sequencetopology',\
'sequenceurl','fnindex_accession','fnindex_id','xreflsid','gi']
summaryItems = [(fieldName, self.databaseFields[fieldName]) for fieldName in summaryFieldNames if self.databaseFields[fieldName] != None]
sequenceItems = [('seqstring',self.databaseFields['seqstring'])]
# get sequence features
connection = databaseModule.getConnection()
# initialise each analysisfunction and test security
if self.obState['DYNAMIC_ANALYSES'] > 0:
connection=databaseModule.getConnection()
accessibleFunctions = []
for func in self.analysisFunctions:
sequencemodulelogger.info('initialising analysisfunction %s to test access '%func[9])
import analysisModule
funcOb = analysisModule.analysisFunction()
funcOb.initFromDatabase(func[9],connection)
funcOb.initMetadata(connection)
funcOb.initProtections(connection)
funcOb.username = self.username
if funcOb.runSecurityFunctions(resourcename="analysis procedure menu"):
sequencemodulelogger.info('adding accessible function %s'%funcOb.databaseFields['xreflsid'])
accessibleFunctions.append(func)
# create the dynamic data section. Item 11 of each function signature is markup containing
# elements used to acquire user input for the function. The section uses a div element with
# id equal to the datasource id
userInputHTML = ''
if len(accessibleFunctions) > 0:
for func in self.analysisFunctions:
myElements = func[11]
if myElements == None:
myElements = ""
myHeading = ""
if len(myElements) > 0:
myHeading = """
<SPAN STYLE="background-color: #ffffcc; color:orange">Specify Parameters for %s</SPAN>
"""%func[1]
myInput = """
<div id=%s style="display:none">
<form name="form%s" id="form%s">
%s
<p/>
%s
</form>
</div>
"""%(func[9],func[9],func[9],myHeading,myElements)
userInputHTML += myInput
if len(accessibleFunctions) > 0:
selectlisttuples = ["<option value=%s> %s : %s </option>"%(item[9], item[3], item[1]) for item in accessibleFunctions ]
selectlisthtml = """
<tr>
<td colspan=2 align=left>
<script language="JavaScript1.2">
function showAnalysisUserInput(selectelement) {
for(i=0 ; i<selectelement.options.length; i++) {
if(selectelement.options[i].selected) {
document.getElementById(selectelement.options[i].value).style.display = 'block';
}
else {
document.getElementById(selectelement.options[i].value).style.display = 'none';
}
}
}
</script>
<font size="-1"> (to select multiple analyses press the control key and click. To select a block use the shift key and click) </font> <p/>
<select name="analyses" id="analyses" multiple size=4 onchange=showAnalysisUserInput(this)>
"""\
+reduce(lambda x,y:x+y+'\n',selectlisttuples,'')\
+ """
</select>
"""\
+ userInputHTML \
+ """
<p/>
<input value="Run Analyses" type="button" id="runanalyses" onclick='multipost("%s","analyses","fetchedanalyses","context=%s&obid=%s&functioninstance=","%s","%s",this)'></p>
<p/>
</td>
</tr>
"""%(self.analysisfetcher,context,self.databaseFields['obid'],self.waitURL,self.waiter)
table += """
<tr> <td colspan="2" class=tableheading>
%s
</td>
</tr>
"""%"Run selected analyses"
table += selectlisthtml
table += """
<tr>
<td>
<div id="fetchedanalyses">
</div>
</td>
</tr>
"""
#table += getSequenceAnnotationBundle(connection, self.databaseFields['obid'], obtype='biosequenceob', usercontext=context, fetcher=self.fetcher,\
# imagepath=self.imagepath, tempimageurl=self.tempimageurl,\
# panelTitle1="Database Search Results",panelTitle2="and Gene Ontology Associations",\
# bundleType = "DBHITS,GO",databaseSearchList = [ 93588, 93591, 93604, 93650, 93974, 94282], topHits = 3, \
# databaseQueryID = self.databaseFields['obid'])
mydisplayhtml = ''
if self.obState['DYNAMIC_DISPLAYS'] > 0:
displayFunctionAccess = True
if self.obState['SECURITY_POLICIES'] > 0:
displayFunctionAccess = self.runSecurityFunctions(context,"display functions")
#mapAccess = False
if displayFunctionAccess:
sequencemodulelogger.info('running non-virtual display functions')
for displayFunction in self.displayFunctions:
# exclude virtual functions - these will be instantiated in specific contexts or subclasses
if displayFunction[7] == None:
sequencemodulelogger.info('running %s'%displayFunction[0])
mydisplayhtml += eval(displayFunction[0])
#table += mydisplayhtml # - now add after summary
featureCursor = connection.cursor()
sql = """
select bsf.xreflsid,featuretype,featurestrand,featurestart,featurestop,featurecomment,evidence
from
biosequencefeaturefact bsf join biosequenceob bso
on
bso.obid = %s and
bsf.biosequenceob = bso.obid
order by
featuretype
"""%self.databaseFields['obid']
featureCursor.execute(sql)
rows = featureCursor.fetchall()
fieldNames = [item[0].lower() for item in featureCursor.description]
# get columnAliases for features
featureHeadings = getColumnAliases(connection,'biosequencefeaturefact')
if featureHeadings == None:
featureHeadings = dict(zip(fieldNames,fieldNames))
for field in fieldNames:
if field not in featureHeadings:
featureHeadings[field] = field
if featureCursor.rowcount > 0:
featureRows = """
<tr> <td colspan="2" class=inside>
<table class=inside>
<tr>
<td class=tableheading colspan="%s"><br/>
Feature Table
</td>
</tr>
"""%len(fieldNames)
featureRows += "<tr>" + reduce(lambda x,y:x+y, ['<td class=fieldname>'+featureHeadings[fieldName]+'</td>\n' \
for fieldName in fieldNames ]) + "</tr>"
if len(rows) <= self.MAX_INLINE_FEATURE_COUNT:
for feature in rows:
featureRow = "<tr>" + reduce(lambda x,y:x+y, ['<td class=fieldvalue>'+str(featureAttribute)+'</td>\n' \
for featureAttribute in feature ]) + "</tr>"
featureRows += featureRow
else:
featureRow = "<tr><td colspan=%s>"%len(fieldNames) + "( ** Too many features - features only displayed inline if there are less than %s : click the Info icon on the Sequence Feature link below to download features ** )"%self.MAX_INLINE_FEATURE_COUNT + "</td></tr>"
featureRows += featureRow
featureRows += "</table></td></tr>"
else:
featureRows = ""
featureCursor.close()
# get vectors and primers
labCursor = connection.cursor()
sql = """
select lr.xreflsid,lr.resourcename, lr.resourcetype, lr.resourcesequence,lr.obid
from
((biosequenceob bs join sequencingfunction sf
on bs.obid = %s and sf.biosequenceob = bs.obid)
join labresourceob lr on lr.obid = sf.labresourceob)
union
select lr.xreflsid,lr.resourcename, lr.resourcetype, lr.resourcesequence, lr.obid
from
(((biosequenceob bs join sequencingfunction sf
on bs.obid = %s and sf.biosequenceob = bs.obid)
join labresourcelistmembershiplink lrl on lrl.labresourcelist = sf.labresourcelist) join
labresourceob lr on lr.obid = lrl.labresourceob)
"""%(self.databaseFields['obid'],self.databaseFields['obid'])
labCursor.execute(sql)
rows = labCursor.fetchall()
fieldNames = [item[0].lower() for item in labCursor.description if item[0].lower() != 'obid']
# modify the data rows - hyperlink the lsids and get rid of the obid
rows = [ ('<a href=' + self.fetcher + "?context=%s&obid=%s&target=ob"%(context,item[4]) + '>%s</a>'%item[0], item[1],item[2], item[3]) for item in rows ]
# get column aliases for lab resources
labHeadings = {
'xreflsid' : 'LSID',
'resourcename' : 'Name',
'resourcetype' : 'Vector/Primer',
'resourcesequence' : 'Sequence'
}
if labCursor.rowcount > 0:
labRows = """
<tr> <td colspan="2" class=inside>
<table class=inside>
<tr>
<td class=tableheading colspan="%s"> <br/>
Vectors / Primers Table
</td>
</tr>
"""%len(fieldNames)
labRows += "<tr>" + reduce(lambda x,y:x+y, ['<td class=fieldname>'+labHeadings[fieldName]+'</td>\n' \
for fieldName in fieldNames ]) + "</tr>"
for resource in rows:
labRow = "<tr>" + reduce(lambda x,y:x+y, ['<td class=fieldvalue>'+str(resourceAttribute)+'</td>\n' \
for resourceAttribute in resource ]) + "</tr>"
labRows += labRow
labRows += "</table></td></tr>"
else:
labRows = ""
labCursor.close()
# get lab resource rows
#FieldItems = [item for item in self.databaseFields.items() if not isinstance(item[1],ListType)]
summaryRows = reduce(lambda x,y:x+y, ['<tr><td class=fieldname>'+self.getColumnAlias(key)+'</td><td class=fieldvalue>'+tidyout(str(value), 80, 1,'<br/>')[0]+'</td></tr>\n' \
for key,value in summaryItems if self.getColumnAlias(key) != None])
seqRows = reduce(lambda x,y:x+y, ['<tr><td class=fieldname>'+self.getColumnAlias(key)+'</td><td class=fieldvalue>'+tidyout(str(value), 80, 1,'<br/>')[0]+'</td></tr>\n' \
for key,value in sequenceItems if self.getColumnAlias(key) != None])
summaryRows = '<tr><td class=inside colspan="2"><table class=inside border="0">' + summaryRows + seqRows + featureRows + labRows + '</table></td></tr>'
# see if there is any expression information available for which we can draw a map
# Note that there is now a displayProcedure that may be dynamically attached - at soem point
# we should probably remove the following hard-coded expression display, and use the
# dnamically attached display procedure
sql = """
select
expressionmapname ,
expressionmaplocus ,
expressionamount
from
geneticexpressionfact
where
biosequenceob = %(obid)s
order by
expressionmapname
"""
sequencemodulelogger.info("executing %s"%sql%self.databaseFields)
expressionCursor=connection.cursor()
expressionCursor.execute(sql,self.databaseFields)
rows = expressionCursor.fetchall()
expressionmaps={}
for row in rows:
if row[0] not in expressionmaps:
expressionmaps[row[0]] = {}
expressionmaps[row[0]].update( {
row[1] : row[2]
})
sequencemodulelogger.info("expression maps : %s"%str(expressionmaps))
myGraphHTML = ""
# for each map , obtain map information including drawing instructions. To have maps drawn,
# an update like this is needed :
#insert into ontologyfact(ontologyob,factnamespace,attributename,attributevalue)
#select obid, 'Display Settings','Expression Graph Type','Bar Graph'
#from ontologyob where ontologyname = 'My Tissue Libaries';
for expressionmap in expressionmaps.keys():
sql = """
select
obid,
attributevalue
from
ontologyob o join ontologyfact otf on
o.ontologyname = %(mapname)s and
otf.ontologyob = o.obid and
otf.factnamespace = 'Display Settings' and
otf.attributename = 'Expression Graph Type'
"""
sequencemodulelogger.info("executing %s"%sql%{ 'mapname' : expressionmap })
expressionCursor.execute(sql,{ 'mapname' : expressionmap })
rows = expressionCursor.fetchall()
if expressionCursor.rowcount != 1:
continue
sequencemodulelogger.info("Expression map info : %s"%str(rows))
graphType = rows[0][1]
# get the expression data
sql = """
select
termname,
termdescription,
obid
from
ontologytermfact
where
ontologyob = %s
"""%rows[0][0]
sequencemodulelogger.info("executing %s"%sql)
expressionCursor.execute(sql)
rows = expressionCursor.fetchall()
mapDomainDict = dict(zip( [row[0] for row in rows], [(row[1],row[2]) for row in rows] ) )
sequencemodulelogger.info("map domain : %s"%str(mapDomainDict))
# we only support a bar graph at the moment
# prepare the arguments to the imageModule method for drawing a bar graph
mapData=[]
for mapDomainItem in mapDomainDict.keys():
dataTuple = [0,mapDomainDict[mapDomainItem][0],self.fetcher + "?context=%s&obid=%s&target=ob"%(context,mapDomainDict[mapDomainItem][1]),mapDomainItem]
#sequencemodulelogger.info("checking if %s in %s"%(mapDomainItem,str(expressionmap)))
if mapDomainItem in expressionmaps[expressionmap]:
dataTuple[0] = expressionmaps[expressionmap][mapDomainItem]
dataTuple = tuple(dataTuple)
mapData.append(dataTuple)
sequencemodulelogger.info(str(mapData))
#graphImageFile = makeBarGraph("c:/temp/",mapData,\
#currenttuple=None,label1="Tissue Expression for",label2=self.databaseFields['sequencename'],\
#barwidth=20,colourScheme=0)
(myGraphName,myGraphMap) = makeBarGraph(imageDirectory=self.imagepath,datatuples=mapData,currenttuple=None,label1="Tissue Expression for",\
label2=self.databaseFields['sequencename'],\
barwidth=15,colourScheme=0)
myGraphHTML= """
<tr>
<td colspan=2 align=center>
<p/>
<img src="%s%s" halign="center" usemap="#%s" border="0"/>
<p/>
%s
</td>
</tr>
"""
myGraphHTML = myGraphHTML%(self.tempimageurl,myGraphName,myGraphName.split('.')[0],myGraphMap)
expressionCursor.close()
connection.close()
#------------------------- end expression maps section ---------------------------
sequencemodulelogger.info('listing fields')
table +="""
<TH colspan="2" style="BACKGROUND: silver ; color:white; font-size:16pt" align=middle>Basic Sequence Details</TH>
"""
table += summaryRows
if len(myGraphHTML) > 0:
table += """
<tr> <td colspan="2" class=inside>
<table class=inside>
<tr>
<td class=tableheading> <br/>
Expression Maps
</td>
</tr>
"""
table += myGraphHTML + mydisplayhtml
return table
def myHTMLSummary(self, table, width=800, context='default'):
""" descendants of the ob class will usually override this method rather than the
entire asHTMLRows method - this method supplies the contents of the summary
panel
"""
FieldItems = [
item for item in self.databaseFields.items()
if not isinstance(item[1], ListType)
]
ListItems = [
item for item in self.databaseFields.items()
if isinstance(item[1], ListType) and len(item[1]) > 0
]
ListDictionaryItems = [
item for item in ListItems if isinstance(item[1][0], DictType)
]
ListOtherItems = [
item for item in ListItems if not isinstance(item[1][0], DictType)
]
nonSystemFieldRows = reduce(lambda x,y:x+y, ['<tr><td class=fieldname>'+self.getColumnAlias(key)+'</td><td class=fieldvalue>'+tidyout(str(value), 80, 1,'<br/>')[0]+'</td></tr>\n' \
for key,value in FieldItems if not key in ( \
'obid','obtypeid','createddate','createdby','lastupdateddate',\
'lastupdatedby','checkedout','checkedoutby','checkoutdate','obkeywords','statuscode') and self.getColumnAlias(key) != None])
nonSystemFieldRows = '<tr><td class=inside colspan="2"><table class=inside border="0">' + nonSystemFieldRows + '</table></td></tr>'
# Format output for values that are lists of dictionaries
# in the next line, the inner reduction concatenates the keys and values for a dictionary - e.g. a single
# function or location , for a gene object.
# the next reduction out concatnates these for each dictionary in the list (i.e. each location, function or whatever etc)
# the final reduction concatenates each category name with all the above - e.g function : location : etc
ListDictionaryRows = ''
if len(ListDictionaryItems) > 0:
ListDictionaryRows = reduce(lambda x,y:x+y, ['<tr><td><b>'+key +'</b></td><td>'+\
'<table>'+ \
reduce(lambda x,y:x+y , [ \
reduce(lambda x,y:x+y, ['<tr><td><i>'+nestedkey+'</i></td><td><b>'+\
str(nestedvalue)+'</b></td></tr>\n' \
for nestedkey,nestedvalue in nestedDict.items()]) + '<p/>' \
for nestedDict in value ]) + \
'</table>' + \
'</td></tr>\n' \
for key,value in ListDictionaryItems])
ListOtherRows = ''
if len(ListOtherItems) > 0:
ListOtherRows = reduce(lambda x, y: x + y, [
'<tr><td>' + key + '</td><td>' + str(value) + '</td></tr>\n'
for key, value in ListOtherItems
])
# un virtual display functions - these are now obtained as part of object initialisation
#sql = """
#select
# df.invocation,
# df.functioncomment,
# dp.xreflsid,
# dp.procedurename,
# ds.datasourcename,
# ds.physicalsourceuri,
# ds.datasourcetype,
# df.voptypeid
#from
# (displayfunction df join displayprocedureob dp on
# df.displayprocedureob = dp.obid) left outer join
# datasourceob ds on ds.obid = df.datasourceob
#where
# df.ob = %(labresourceob)s and
# df.voptypeid is not null
#"""
#labresourcemodulelogger.info('executing SQL to retrieve dynamic display functions : %s'%str(sql%self.databaseFields))
connection = databaseModule.getConnection()
#displayCursor = connection.cursor()
#displayCursor.execute(sql,self.databaseFields)
#displayFunctions = displayCursor.fetchall()
#labresourcemodulelogger.info(str(displayFunctions))
# test code :
#myGraphHTML = eval('getGenepixThumbnailDisplay(jpegfilename="C:/working/zaneta/9072 ratio 1 mid scan .jpg",xy=re.split("\t",self.databaseFields["rawdatarecord"])[5:7],usercontext=context,fetcher=self.fetcher, imagepath=self.imagepath, tempimageurl=self.tempimageurl, sectionheading="Spot Images" ,pixelsize=10,xyoffset=(780, 12780),thumbcount=3,thumbdimensions=(20,20),zoomincrement=50)')
#table += myGraphHTML
for displayFunction in self.virtualDisplayFunctions:
labresourcemodulelogger.info(
"executing following display code : %s" % displayFunction[0])
myGraphHTML = eval(displayFunction[0])
table += myGraphHTML
labresourcemodulelogger.info(
'checking whether graphic can be made of expression values')
# if the data source type for this microarray experiment is GALFile then we try
# to graph the logratios and intensities from all observations of this spot. If it is
# CSV from Affy CEL File the we try to graph Affy stats
connection = databaseModule.getConnection()
graphCursor = connection.cursor()
sql = """
select
d.datasourcetype
from
((microarrayspotfact msf join labresourceob lr
on lr.obid = msf.labresourceob ) join
importfunction if on if.ob = lr.obid ) join
datasourceob d on d.obid = if.datasourceob
where
msf.obid = %s
""" % self.databaseFields['obid']
labresourcemodulelogger.info("executing %s" % str(sql))
graphCursor.execute(sql)
record = graphCursor.fetchone()
if graphCursor.rowcount == 1:
if record[0] in [
'GALFile', 'AgResearchArrayExtract1', 'GALFile_noheader'
]:
# get and graph the log ratios from all observations related to this spot.
sql = """
select
mo.gpr_logratio,
mo.xreflsid,
mo.obid,
(mo.gpr_dye1foregroundmean + mo.gpr_dye2foregroundmean)/2.0 as averagefg,
ges.xreflsid as studylsid,
substr(ges.studydescription,1,30) as studydescription
from
geneexpressionstudy ges join microarrayobservation mo
on
ges.obid = mo.microarraystudy
where
microarrayspotfact = %s order by
mo.xreflsid""" % self.databaseFields['obid']
labresourcemodulelogger.info("executing %s" % str(sql))
graphCursor.execute(sql)
datatuples = graphCursor.fetchall()
labresourcemodulelogger.info("..done executing have data")
observationids = [item[2] for item in datatuples]
# each tuple contain a fetch URL - initialise this
logratiodatatuples = [ (item[0],item[1], self.fetcher + "?context=%s&obid=%s&target=ob"%(context,item[2])) \
for item in datatuples ]
#datatuples = [(item[0],item[1]) for item in datatuples]
if graphCursor.rowcount > 0:
# construct a multi-select list that can be used to select thumbnails to display
# (if they are available)
selectlisttuples = [
"<option value=%s> %s %s (%s)</option>" %
(item[2], item[4], item[5], item[2])
for item in datatuples
]
selectlisthtml = """
<tr>
<td colspan=2 align=center>
<!-- <input value="Go" type="button" onclick='multipost("/cgi-bin/sheepgenomics/simple-ajax-example.py","experiments","fetcheddisplays","displayprocedure=34&option=")'></p> -->
<font size="-1"> (to select multiple experiments press the control key and click. To select a block use the shift key and click) </font> <p/>
<select name="experiments" id="experiments" multiple size=4>
"""\
+reduce(lambda x,y:x+y+'\n',selectlisttuples,'')\
+ """
</select>
<p/>
<input value="Retrieve spot thumbnails for selected experiments" type="button" id="getspots" onclick='multipost("%s","experiments","fetcheddisplays","context=%s&displayprocedure=displayProcedures.getGenepixThumbnailDisplay&obid=","%s","%s",this)'></p>
<p/>
</td>
</tr>
"""%(self.displayfetcher,context,self.waitURL,self.waiter)
#<input value="Retrieve spot thumbnails for selected experiments" type="button" onclick='multipost("/cgi-bin/sheepgenomics/fetchDisplay.py","experiments","fetcheddisplays","context=%s&displayprocedure=displayProcedures.getGenepixThumbnailDisplay&obid=")'></p>
(myGraphName,
myGraphMap) = makeBarGraph(imageDirectory=self.imagepath,
datatuples=logratiodatatuples,
currenttuple=None,
label1="All raw LogRatios",
label2="for this spot",
barwidth=5)
myGraphHTML = """
<tr>
<td colspan=2 align=center>
<p/>
<img src="%s%s" halign="center" usemap="#%s" border="0"/>
<p/>
%s
</td>
</tr>
"""
myGraphHTML = myGraphHTML % (
self.tempimageurl, myGraphName,
myGraphName.split('.')[0], myGraphMap)
table += """
<tr> <td colspan="2" class=tableheading>
%s
</td>
</tr>
""" % "All observed raw logratios and intensities for this gene, in experiments using this chip"
table += myGraphHTML
# graph the intensities
intensitydatatuples = [ (int(item[3]),item[1], self.fetcher + "?context=%s&obid=%s&target=ob"%(context,item[2])) \
for item in datatuples ]
#datatuples = [(item[0],item[1]) for item in datatuples]
if graphCursor.rowcount > 0:
(myGraphName, myGraphMap) = makeBarGraph(
imageDirectory=self.imagepath,
datatuples=intensitydatatuples,
currenttuple=None,
label1="All average foreground",
label2="intensities for this spot",
barwidth=5)
myGraphHTML = """
<tr>
<td colspan=2 align=center>
<p/>
<img src="%s%s" halign="center" usemap="#%s" border="0"/>
<p/>
%s
</td>
</tr>
"""
myGraphHTML = myGraphHTML % (
self.tempimageurl, myGraphName,
myGraphName.split('.')[0], myGraphMap)
table += myGraphHTML
table += """
<tr> <td colspan="2" class=tableheading>
%s
</td>
</tr>
""" % "Review selected experiments"
table += selectlisthtml
table += """
<tr>
<td>
<div id="fetcheddisplays">
</div>
</td>
</tr>
"""
# we now see whether there are any normalised values that we should graph.
# the available normalised values are stored in an ontology called
# MICROARRAY_NORMALISED_VALUES.
sql = """
select otf.termName , otf.unitname from
ontologyob ot join ontologytermfact otf
on otf.ontologyob = ot.obid
where
ot.ontologyname = 'MICROARRAY_NORMALISED_VALUES'
order by otf.termName
"""
labresourcemodulelogger.info(
"getting normalised data point names using %s" % sql)
graphCursor.execute(sql)
datapoints = graphCursor.fetchall()
graphHTML = ""
for (datapoint, datatype) in datapoints:
# obtain the data points - we re-use the above array of data tuples, since they
# contain the correct tooltips and urls - just change the data point value
skipdatapoint = False
for iobservation in range(0, len(observationids)):
sql = """
select case when attributeValue is null then '' else attributeValue end
from microarrayobservationfact
where
microarrayobservation = %s and
factNameSpace = 'NORMALISED VALUE' and
attributeName = '%s'
""" % (observationids[iobservation], datapoint)
labresourcemodulelogger.info(
"getting normalised data points using %s" % sql)
graphCursor.execute(sql)
datapointvalue = graphCursor.fetchone()
if graphCursor.rowcount == 1:
if datapointvalue[0] == None:
datatuples[iobservation] = \
(None, datatuples[iobservation][1], logratiodatatuples[iobservation][2])
elif len(datapointvalue[0]) == 0:
datatuples[iobservation] = \
(None, datatuples[iobservation][1], logratiodatatuples[iobservation][2])
else:
datatuples[iobservation] = \
(float(datapointvalue[0]), datatuples[iobservation][1], logratiodatatuples[iobservation][2])
else:
skipdatapoint = True
labresourcemodulelogger.info(
"skipping data point - query returned no rows")
break # we got nothing for this observation - incomplete dataset, give up
# if all values missing, skip the whole data point
notMissing = [
datatuple[0] for datatuple in datatuples
if datatuple[0] != None
]
if len(notMissing) == 0:
skipdatapoint = True
if not skipdatapoint:
(myGraphName, myGraphMap) = makeBarGraph(
imageDirectory=self.imagepath,
datatuples=datatuples,
currenttuple=None,
label1="Normalisation:",
label2=datapoint,
barwidth=5)
myGraphHTML = """
<tr>
<td colspan=2 align=center>
<p/>
<img src="%s%s" halign="center" usemap="#%s" border="0"/>
<p/>
%s
</td>
</tr>
"""
myGraphHTML = myGraphHTML % (
self.tempimageurl, myGraphName,
myGraphName.split('.')[0], myGraphMap)
graphHTML += myGraphHTML
table += """
<tr> <td colspan="2" class=tableheading>
%s
</td>
</tr>
""" % "All normalised values available for this gene, in experiments using this chip"
table += graphHTML
# Affy arrays are imported in several steps, and the array itself is not linked to
# a source file. We can distinguish Affy arrays because the lsid of this spot will begin with
# affymetrix
elif re.search('(\.)*Affymetrix\.', self.databaseFields['xreflsid'],
re.IGNORECASE) != None:
labresourcemodulelogger.info(
"attempting to display Affy experiment results")
sql = """
select
mo.affy_meanpm,
mo.affy_meanmm,
mo.affy_stddevpm,
mo.affy_stddevmm,
mo.affy_count,
mo.xreflsid,
mo.obid
from
microarrayobservation mo
where
microarrayspotfact = %s order by
mo.xreflsid""" % self.databaseFields['obid']
labresourcemodulelogger.info("executing %s" % str(sql))
graphCursor.execute(sql)
datatuples = graphCursor.fetchall()
labresourcemodulelogger.info("..done executing have data")
observationids = [item[6] for item in datatuples]
# each tuple contain a fetch URL - initialise this
pmdatatuples = [ (item[0],item[5], self.fetcher + "?context=%s&obid=%s&target=ob"%(context,item[6])) \
for item in datatuples ]
#datatuples = [(item[0],item[1]) for item in datatuples]
if graphCursor.rowcount > 0:
# PM means
(myGraphName,
myGraphMap) = makeBarGraph(imageDirectory=self.imagepath,
datatuples=pmdatatuples,
currenttuple=None,
label1="All Probeset mean PM",
label2="for this probeset",
barwidth=5)
myGraphHTML = """
<tr>
<td colspan=2 align=center>
<p/>
<img src="%s%s" halign="center" usemap="#%s" border="0"/>
<p/>
%s
</td>
</tr>
"""
myGraphHTML = myGraphHTML % (self.tempimageurl, myGraphName,
myGraphName.split('.')[0],
myGraphMap)
table += """
<tr> <td colspan="2" class=tableheading>
%s
</td>
</tr>
""" % "All observed probeset means and standard deviations for this probeset, across experiments using this chip"
table += myGraphHTML
#PM standard deviations
pmdatatuples = [ (item[2],item[5], self.fetcher + "?context=%s&obid=%s&target=ob"%(context,item[6])) \
for item in datatuples ]
(myGraphName,
myGraphMap) = makeBarGraph(imageDirectory=self.imagepath,
datatuples=pmdatatuples,
currenttuple=None,
label1="All Probeset stddev PM",
label2="for this probeset",
barwidth=5)
myGraphHTML = """
<tr>
<td colspan=2 align=center>
<p/>
<img src="%s%s" halign="center" usemap="#%s" border="0"/>
<p/>
%s
</td>
</tr>
"""
myGraphHTML = myGraphHTML % (self.tempimageurl, myGraphName,
myGraphName.split('.')[0],
myGraphMap)
table += myGraphHTML
graphCursor.close()
if self.obState['DYNAMIC_DISPLAYS'] > 0:
labresourcemodulelogger.info(
'running non-virtual display functions')
for displayFunction in self.displayFunctions:
# exclude virtual functions - these will be instantiated in specific contexts or subclasses
if displayFunction[7] == None:
labresourcemodulelogger.info('running %s' %
displayFunction[0])
myGraphHTML = eval(displayFunction[0])
table += myGraphHTML
labresourcemodulelogger.info('listing dictionaries')
# if we have formatted dictionaries , output these first , they are usually the most interesting
# content of the object
if len(ListDictionaryRows) > 0:
table += ListDictionaryRows
labresourcemodulelogger.info('listing fields')
# next the field rows
table += nonSystemFieldRows
labresourcemodulelogger.info('listing lists')
# next the other lists
if len(ListOtherRows) > 0:
table += ListOtherRows
return table