def gen_sql(self, mode="heatmap"):
if "compiler.mode" in self.params and self.params[
"compiler.mode"] == "scan":
return
log("Writing SQL " + mode)
if not os.path.exists(self.out_dir):
os.makedirs(self.out_dir)
self.id_table = CGIDTable()
for rtype in self.compile_matrix:
for rname in self.compile_matrix[rtype]:
if hasattr(self.compile_matrix[rtype][rname],
"gen_sql_" + mode):
sql_func = getattr(self.compile_matrix[rtype][rname],
"gen_sql_" + mode)
shandle = sql_func(self.id_table)
if shandle is not None:
opath = os.path.join(self.out_dir,
"%s.%s.sql" % (rtype, rname))
ohandle = open(opath, "w")
try:
for line in shandle:
ohandle.write(line)
ohandle.close()
except Exception:
ohandle.close()
os.unlink(opath)
error("Track %s failed" % (rname))
#tell the object to unload data, so we don't continually allocate over the compile
self.compile_matrix[rtype][rname].unload()
def gen_sql(self, mode="heatmap"):
if "compiler.mode" in self.params and self.params[ "compiler.mode" ] == "scan":
return
log( "Writing SQL " + mode )
if not os.path.exists(self.out_dir):
os.makedirs(self.out_dir)
self.id_table = CGIDTable()
for rtype in self.compile_matrix:
for rname in self.compile_matrix[ rtype ]:
if hasattr(self.compile_matrix[ rtype ][ rname ], "gen_sql_" + mode):
sql_func = getattr(self.compile_matrix[ rtype ][ rname ], "gen_sql_" + mode)
shandle = sql_func(self.id_table)
if shandle is not None:
opath = os.path.join( self.out_dir, "%s.%s.sql" % (rtype, rname ) )
ohandle = open( opath, "w" )
try:
for line in shandle:
ohandle.write( line )
ohandle.close()
except Exception:
ohandle.close()
os.unlink(opath)
error("Track %s failed" % (rname))
#tell the object to unload data, so we don't continually allocate over the compile
self.compile_matrix[ rtype ][ rname ].unload()
def gen_sql(self):
if "compiler.mode" in self.params and self.params[ "compiler.mode" ] == "scan":
return
log( "Writing SQL" )
if not os.path.exists(self.out_dir):
os.makedirs(self.out_dir)
self.id_table = CGIDTable()
for rtype in self.compile_matrix:
if issubclass( CGData.get_type( rtype ), CGData.CGSQLObject ):
for rname in self.compile_matrix[ rtype ]:
shandle = self.compile_matrix[ rtype ][ rname ].gen_sql( self.id_table )
if shandle is not None:
ohandle = open( os.path.join( self.out_dir, "%s.%s.sql" % (rtype, rname ) ), "w" )
for line in shandle:
ohandle.write( line )
ohandle.close()
#tell the object to unload data, so we don't continually allocate over the compile
self.compile_matrix[ rtype ][ rname ].unload()
def link_objects(self):
"""
Scan found object records and determine if the data they link to is
avalible
"""
omatrix = {}
for otype in self.set_hash:
if issubclass( CGData.get_type( otype ), CGData.CGGroupMember ):
gmap = {}
for oname in self.set_hash[ otype ]:
oobj = self.set_hash[ otype ][ oname ]
if oobj.get_group() not in gmap:
if issubclass(CGData.get_type(otype), CGData.CGGroupMemberSQL):
gmap[ oobj.get_group() ] = CGData.CGGroupBaseSQL( oobj.get_group() )
else:
gmap[ oobj.get_group() ] = CGData.CGGroupBase( oobj.get_group() )
gmap[ oobj.get_group() ].put( oobj )
omatrix[ otype ] = gmap
else:
omatrix[ otype ] = self.set_hash[ otype ]
# Now it's time to check objects for their dependencies
ready_matrix = {}
for stype in omatrix:
for sname in omatrix[ stype ]:
sobj = omatrix[ stype ][ sname ]
lmap = sobj.get_link_map()
is_ready = True
for ltype in lmap:
if not omatrix.has_key( ltype ):
warn( "%s missing data type %s" % (sname, ltype) )
is_ready = False
else:
for lname in lmap[ ltype ]:
if not omatrix[ltype].has_key( lname ):
warn( "%s %s missing data %s %s" % ( stype, sname, ltype, lname ) )
is_ready = False
if not sobj.is_link_ready():
warn( "%s %s not LinkReady" % ( stype, sname ) )
elif is_ready:
if not stype in ready_matrix:
ready_matrix[ stype ] = {}
ready_matrix[ stype ][ sname ] = sobj
for rtype in ready_matrix:
log( "READY %s: %s" % ( rtype, ",".join(ready_matrix[rtype].keys()) ) )
for dType in ready_matrix:
log("Found %s %d" % (dType, len(ready_matrix[dType])))
merge_children = {}
for merge_type in CGData.MERGE_OBJECTS:
mtype = CGData.get_type( merge_type )
select_types = mtype.typeSet
select_set = {}
try:
for stype in select_types:
select_set[ stype ] = ready_matrix[ stype ]
if stype not in merge_children:
merge_children[stype] = {}
except KeyError:
error("missing data type %s" % (stype) )
continue
mobjlist = self.set_enumerate( mtype, select_set )
for mobj in mobjlist:
if merge_type not in ready_matrix:
ready_matrix[ merge_type ] = {}
for cType in mobj:
merge_children[cType][mobj[cType].get_name()] = True
ready_matrix[ merge_type ][ mobj.get_name() ] = mobj
self.compile_matrix = {}
for sType in ready_matrix:
self.compile_matrix[sType] = {}
for name in ready_matrix[sType]:
if sType not in merge_children or name not in merge_children[sType]:
self.compile_matrix[sType][name] = ready_matrix[sType][name]
log("After Merge")
for dType in ready_matrix:
log("Found %s %d" % (dType, len(self.compile_matrix[dType])))
def set_enumerate( self, merge_type, a, b={} ):
"""
This is an recursive function to enumerate possible sets of elements in the 'a' hash
a is a map of types ('probeMap', 'clinicalMatrix', ...), each of those is a map
of cgBaseObjects that report get_link_map requests
"""
#print "Enter", " ".join( (b[c].get_name() for c in b) )
cur_key = None
for t in a:
if not t in b:
cur_key = t
if cur_key is None:
#make sure selected subgraph is connected
#start by building a graph of connections
#and map of connected nodes
cMap = {}
lMap = {}
for c in b:
n = "%s:%s" % (c, b[c].get_name())
cMap[ n ] = False
lMap[n] = {}
for d in b[c].get_link_map():
for e in b[c].get_link_map()[d]:
m = "%s:%s" % (d,e)
lMap[n][m] = True
#add the first node to the connected set
cMap[ cMap.keys()[0] ] = True
found = True
#continue adding nodes to the connected set, until no more can be found
while found:
found = False
for c in cMap:
if not cMap[c]:
for d in cMap:
if cMap[d]:
if d in lMap[c] or c in lMap[d]:
found = True
cMap[c] = True
cMap[d] = True
#if there are no disconnected nodes, then the subset represents a connected graph,
#and is ready to merge
if cMap.values().count(False) == 0:
#print " ".join( ( "%s:%s:%s" % (c, b[c].get_name(), str(b[c].get_link_map()) ) for c in b) )
log( "Merging %s" % ",".join( ( "%s:%s" %(c,b[c].get_name()) for c in b) ) )
mergeObj = merge_type()
mergeObj.merge( **b )
return [ mergeObj ]
else:
out = []
for i in a[cur_key]:
#print "Trying", cur_key, i
c = copy(b)
sobj = a[cur_key][i] #the object selected to be added next
lmap = sobj.get_link_map()
valid = True
for ltype in lmap:
if ltype in c:
if c[ltype].get_name() not in lmap[ltype]:
#print c[ltype].get_name(), "not in", lmap[ltype]
valid = False
for stype in c:
slmap = c[stype].get_link_map()
for sltype in slmap:
if cur_key == sltype:
if sobj.get_name() not in slmap[sltype]:
#print a[cur_key][i].get_name(), "not in", slmap[sltype]
valid = False
if valid:
c[ cur_key ] = sobj
out.extend( self.set_enumerate( merge_type, a, c ) )
return out
return []
def link_objects(self):
"""
Scan found object records and determine if the data they link to is
avalible
"""
omatrix = {}
for otype in self.set_hash:
if issubclass(CGData.get_type(otype), CGData.CGGroupMember):
gmap = {}
for oname in self.set_hash[otype]:
oobj = self.set_hash[otype][oname]
if oobj.get_group() not in gmap:
gmap[oobj.get_group()] = CGData.CGGroupBase(
oobj.get_group())
gmap[oobj.get_group()].put(oobj)
omatrix[otype] = gmap
else:
omatrix[otype] = self.set_hash[otype]
# Now it's time to check objects for their dependencies
ready_matrix = {}
for stype in omatrix:
for sname in omatrix[stype]:
sobj = omatrix[stype][sname]
lmap = sobj.get_link_map()
is_ready = True
for ltype in lmap:
if not omatrix.has_key(ltype):
warn("%s missing data type %s" % (sname, ltype))
is_ready = False
else:
for lname in lmap[ltype]:
if not omatrix[ltype].has_key(lname):
warn("%s %s missing data %s %s" %
(stype, sname, ltype, lname))
is_ready = False
if not sobj.is_link_ready():
warn("%s %s not LinkReady" % (stype, sname))
elif is_ready:
if not stype in ready_matrix:
ready_matrix[stype] = {}
ready_matrix[stype][sname] = sobj
for rtype in ready_matrix:
log("READY %s: %s" % (rtype, ",".join(ready_matrix[rtype].keys())))
for dType in ready_matrix:
log("Found %s %d" % (dType, len(ready_matrix[dType])))
merge_children = {}
for merge_type in CGData.MERGE_OBJECTS:
mtype = CGData.get_type(merge_type)
select_types = mtype.typeSet
select_set = {}
try:
for stype in select_types:
select_set[stype] = ready_matrix[stype]
if stype not in merge_children:
merge_children[stype] = {}
except KeyError:
error("missing data type %s" % (stype))
continue
mobjlist = self.set_enumerate(mtype, select_set)
for mobj in mobjlist:
if merge_type not in ready_matrix:
ready_matrix[merge_type] = {}
for cType in mobj:
merge_children[cType][mobj[cType].get_name()] = True
ready_matrix[merge_type][mobj.get_name()] = mobj
self.compile_matrix = {}
for sType in ready_matrix:
self.compile_matrix[sType] = {}
for name in ready_matrix[sType]:
if sType not in merge_children or name not in merge_children[
sType]:
self.compile_matrix[sType][name] = ready_matrix[sType][
name]
log("After Merge")
for dType in ready_matrix:
log("Found %s %d" % (dType, len(self.compile_matrix[dType])))
def set_enumerate(self, merge_type, a, b={}):
"""
This is an recursive function to enumerate possible sets of elements in the 'a' hash
a is a map of types ('probeMap', 'clinicalMatrix', ...), each of those is a map
of cgBaseObjects that report get_link_map requests
"""
cur_key = None
for t in a:
if not t in b:
cur_key = t
if cur_key is None:
#make sure selected subgraph is connected
#start by building a graph of connections
#and map of connected nodes
cMap = {}
lMap = {}
for c in b:
n = "%s:%s" % (c, b[c].get_name())
cMap[n] = False
lMap[n] = {}
for d in b[c].get_link_map():
for e in b[c].get_link_map()[d]:
m = "%s:%s" % (d, e)
lMap[n][m] = True
#add the first node to the connected set
cMap[cMap.keys()[0]] = True
found = True
#continue adding nodes to the connected set, until no more can be found
while found:
found = False
for c in cMap:
if not cMap[c]:
for d in cMap:
if cMap[d]:
if d in lMap[c] or c in lMap[d]:
found = True
cMap[c] = True
cMap[d] = True
#if there are no disconnected nodes, then the subset represents a connected graph,
#and is ready to merge
if cMap.values().count(False) == 0:
log("Merging %s" % ",".join(
("%s:%s" % (c, b[c].get_name()) for c in b)))
mergeObj = merge_type()
mergeObj.merge(**b)
return [mergeObj]
else:
out = []
for i in a[cur_key]:
c = copy(b)
sobj = a[cur_key][i] #the object selected to be added next
lmap = sobj.get_link_map()
valid = True
for ltype in lmap:
if ltype in c:
if c[ltype].get_name() not in lmap[ltype]:
valid = False
for stype in c:
slmap = c[stype].get_link_map()
for sltype in slmap:
if cur_key == sltype:
if sobj.get_name() not in slmap[sltype]:
valid = False
if valid:
c[cur_key] = sobj
out.extend(self.set_enumerate(merge_type, a, c))
return out
return []
