def gen_sql(self, mode="heatmap"):
if "compiler.mode" in self.params and self.params[
"compiler.mode"] == "scan":
return
log("Writing SQL " + mode)
if not os.path.exists(self.out_dir):
os.makedirs(self.out_dir)
self.id_table = CGIDTable()
for rtype in self.compile_matrix:
for rname in self.compile_matrix[rtype]:
if hasattr(self.compile_matrix[rtype][rname],
"gen_sql_" + mode):
sql_func = getattr(self.compile_matrix[rtype][rname],
"gen_sql_" + mode)
shandle = sql_func(self.id_table)
if shandle is not None:
opath = os.path.join(self.out_dir,
"%s.%s.sql" % (rtype, rname))
ohandle = open(opath, "w")
try:
for line in shandle:
ohandle.write(line)
ohandle.close()
except Exception:
ohandle.close()
os.unlink(opath)
error("Track %s failed" % (rname))
#tell the object to unload data, so we don't continually allocate over the compile
self.compile_matrix[rtype][rname].unload()
def gen_sql(self, mode="heatmap"):
if "compiler.mode" in self.params and self.params[ "compiler.mode" ] == "scan":
return
log( "Writing SQL " + mode )
if not os.path.exists(self.out_dir):
os.makedirs(self.out_dir)
self.id_table = CGIDTable()
for rtype in self.compile_matrix:
for rname in self.compile_matrix[ rtype ]:
if hasattr(self.compile_matrix[ rtype ][ rname ], "gen_sql_" + mode):
sql_func = getattr(self.compile_matrix[ rtype ][ rname ], "gen_sql_" + mode)
shandle = sql_func(self.id_table)
if shandle is not None:
opath = os.path.join( self.out_dir, "%s.%s.sql" % (rtype, rname ) )
ohandle = open( opath, "w" )
try:
for line in shandle:
ohandle.write( line )
ohandle.close()
except Exception:
ohandle.close()
os.unlink(opath)
error("Track %s failed" % (rname))
#tell the object to unload data, so we don't continually allocate over the compile
self.compile_matrix[ rtype ][ rname ].unload()
def link_objects(self):
"""
Scan found object records and determine if the data they link to is
avalible
"""
omatrix = {}
for otype in self.set_hash:
if issubclass( CGData.get_type( otype ), CGData.CGGroupMember ):
gmap = {}
for oname in self.set_hash[ otype ]:
oobj = self.set_hash[ otype ][ oname ]
if oobj.get_group() not in gmap:
if issubclass(CGData.get_type(otype), CGData.CGGroupMemberSQL):
gmap[ oobj.get_group() ] = CGData.CGGroupBaseSQL( oobj.get_group() )
else:
gmap[ oobj.get_group() ] = CGData.CGGroupBase( oobj.get_group() )
gmap[ oobj.get_group() ].put( oobj )
omatrix[ otype ] = gmap
else:
omatrix[ otype ] = self.set_hash[ otype ]
# Now it's time to check objects for their dependencies
ready_matrix = {}
for stype in omatrix:
for sname in omatrix[ stype ]:
sobj = omatrix[ stype ][ sname ]
lmap = sobj.get_link_map()
is_ready = True
for ltype in lmap:
if not omatrix.has_key( ltype ):
warn( "%s missing data type %s" % (sname, ltype) )
is_ready = False
else:
for lname in lmap[ ltype ]:
if not omatrix[ltype].has_key( lname ):
warn( "%s %s missing data %s %s" % ( stype, sname, ltype, lname ) )
is_ready = False
if not sobj.is_link_ready():
warn( "%s %s not LinkReady" % ( stype, sname ) )
elif is_ready:
if not stype in ready_matrix:
ready_matrix[ stype ] = {}
ready_matrix[ stype ][ sname ] = sobj
for rtype in ready_matrix:
log( "READY %s: %s" % ( rtype, ",".join(ready_matrix[rtype].keys()) ) )
for dType in ready_matrix:
log("Found %s %d" % (dType, len(ready_matrix[dType])))
merge_children = {}
for merge_type in CGData.MERGE_OBJECTS:
mtype = CGData.get_type( merge_type )
select_types = mtype.typeSet
select_set = {}
try:
for stype in select_types:
select_set[ stype ] = ready_matrix[ stype ]
if stype not in merge_children:
merge_children[stype] = {}
except KeyError:
error("missing data type %s" % (stype) )
continue
mobjlist = self.set_enumerate( mtype, select_set )
for mobj in mobjlist:
if merge_type not in ready_matrix:
ready_matrix[ merge_type ] = {}
for cType in mobj:
merge_children[cType][mobj[cType].get_name()] = True
ready_matrix[ merge_type ][ mobj.get_name() ] = mobj
self.compile_matrix = {}
for sType in ready_matrix:
self.compile_matrix[sType] = {}
for name in ready_matrix[sType]:
if sType not in merge_children or name not in merge_children[sType]:
self.compile_matrix[sType][name] = ready_matrix[sType][name]
log("After Merge")
for dType in ready_matrix:
log("Found %s %d" % (dType, len(self.compile_matrix[dType])))
def link_objects(self):
"""
Scan found object records and determine if the data they link to is
avalible
"""
omatrix = {}
for otype in self.set_hash:
if issubclass(CGData.get_type(otype), CGData.CGGroupMember):
gmap = {}
for oname in self.set_hash[otype]:
oobj = self.set_hash[otype][oname]
if oobj.get_group() not in gmap:
gmap[oobj.get_group()] = CGData.CGGroupBase(
oobj.get_group())
gmap[oobj.get_group()].put(oobj)
omatrix[otype] = gmap
else:
omatrix[otype] = self.set_hash[otype]
# Now it's time to check objects for their dependencies
ready_matrix = {}
for stype in omatrix:
for sname in omatrix[stype]:
sobj = omatrix[stype][sname]
lmap = sobj.get_link_map()
is_ready = True
for ltype in lmap:
if not omatrix.has_key(ltype):
warn("%s missing data type %s" % (sname, ltype))
is_ready = False
else:
for lname in lmap[ltype]:
if not omatrix[ltype].has_key(lname):
warn("%s %s missing data %s %s" %
(stype, sname, ltype, lname))
is_ready = False
if not sobj.is_link_ready():
warn("%s %s not LinkReady" % (stype, sname))
elif is_ready:
if not stype in ready_matrix:
ready_matrix[stype] = {}
ready_matrix[stype][sname] = sobj
for rtype in ready_matrix:
log("READY %s: %s" % (rtype, ",".join(ready_matrix[rtype].keys())))
for dType in ready_matrix:
log("Found %s %d" % (dType, len(ready_matrix[dType])))
merge_children = {}
for merge_type in CGData.MERGE_OBJECTS:
mtype = CGData.get_type(merge_type)
select_types = mtype.typeSet
select_set = {}
try:
for stype in select_types:
select_set[stype] = ready_matrix[stype]
if stype not in merge_children:
merge_children[stype] = {}
except KeyError:
error("missing data type %s" % (stype))
continue
mobjlist = self.set_enumerate(mtype, select_set)
for mobj in mobjlist:
if merge_type not in ready_matrix:
ready_matrix[merge_type] = {}
for cType in mobj:
merge_children[cType][mobj[cType].get_name()] = True
ready_matrix[merge_type][mobj.get_name()] = mobj
self.compile_matrix = {}
for sType in ready_matrix:
self.compile_matrix[sType] = {}
for name in ready_matrix[sType]:
if sType not in merge_children or name not in merge_children[
sType]:
self.compile_matrix[sType][name] = ready_matrix[sType][
name]
log("After Merge")
for dType in ready_matrix:
log("Found %s %d" % (dType, len(self.compile_matrix[dType])))