Пример #1
0
 def gen_sql(self, mode="heatmap"):
     if "compiler.mode" in self.params and self.params[
             "compiler.mode"] == "scan":
         return
     log("Writing SQL " + mode)
     if not os.path.exists(self.out_dir):
         os.makedirs(self.out_dir)
     self.id_table = CGIDTable()
     for rtype in self.compile_matrix:
         for rname in self.compile_matrix[rtype]:
             if hasattr(self.compile_matrix[rtype][rname],
                        "gen_sql_" + mode):
                 sql_func = getattr(self.compile_matrix[rtype][rname],
                                    "gen_sql_" + mode)
                 shandle = sql_func(self.id_table)
                 if shandle is not None:
                     opath = os.path.join(self.out_dir,
                                          "%s.%s.sql" % (rtype, rname))
                     ohandle = open(opath, "w")
                     try:
                         for line in shandle:
                             ohandle.write(line)
                         ohandle.close()
                     except Exception:
                         ohandle.close()
                         os.unlink(opath)
                         error("Track %s failed" % (rname))
                 #tell the object to unload data, so we don't continually allocate over the compile
                 self.compile_matrix[rtype][rname].unload()
Пример #2
0
 def gen_sql(self, mode="heatmap"):
     if "compiler.mode" in self.params and self.params[ "compiler.mode" ] == "scan":
         return
     log( "Writing SQL " + mode  )     
     if not os.path.exists(self.out_dir):
         os.makedirs(self.out_dir)
     self.id_table = CGIDTable()
     for rtype in self.compile_matrix:
         for rname in self.compile_matrix[ rtype ]:
             if hasattr(self.compile_matrix[ rtype ][ rname ], "gen_sql_" + mode):
                 sql_func = getattr(self.compile_matrix[ rtype ][ rname ], "gen_sql_" + mode)
                 shandle = sql_func(self.id_table)
                 if shandle is not None:
                     opath = os.path.join( self.out_dir, "%s.%s.sql" % (rtype, rname ) )
                     ohandle = open( opath, "w" )
                     try:
                         for line in shandle:
                             ohandle.write( line )
                         ohandle.close()
                     except Exception:
                         ohandle.close()
                         os.unlink(opath)
                         error("Track %s failed" % (rname))
                 #tell the object to unload data, so we don't continually allocate over the compile
                 self.compile_matrix[ rtype ][ rname ].unload()
Пример #3
0
    def link_objects(self):
        """
        Scan found object records and determine if the data they link to is
        avalible
        """
        omatrix = {}
        for otype in self.set_hash:
            if issubclass( CGData.get_type( otype ), CGData.CGGroupMember ):
                gmap = {}
                for oname in self.set_hash[ otype ]:
                    oobj = self.set_hash[ otype ][ oname ]
                    if oobj.get_group() not in gmap:
                        if issubclass(CGData.get_type(otype), CGData.CGGroupMemberSQL):
                            gmap[ oobj.get_group() ] = CGData.CGGroupBaseSQL( oobj.get_group() )
                        else:
                            gmap[ oobj.get_group() ] = CGData.CGGroupBase( oobj.get_group() )
                    gmap[ oobj.get_group() ].put( oobj )
                omatrix[ otype ] = gmap
            else:
                omatrix[ otype ] = self.set_hash[ otype ]
        
        # Now it's time to check objects for their dependencies
        ready_matrix = {}
        for stype in omatrix:
            for sname in omatrix[ stype ]:
                sobj = omatrix[ stype ][ sname ]
                lmap = sobj.get_link_map()
                is_ready = True
                for ltype in lmap:
                    if not omatrix.has_key( ltype ):
                        warn( "%s missing data type %s" % (sname, ltype) )
                        is_ready = False
                    else:
                        for lname in lmap[ ltype ]:
                            if not omatrix[ltype].has_key( lname ):
                                warn( "%s %s missing data %s %s" % ( stype, sname, ltype, lname ) )
                                is_ready = False
                if not sobj.is_link_ready():
                    warn( "%s %s not LinkReady" % ( stype, sname ) )
                elif is_ready:
                    if not stype in ready_matrix:
                        ready_matrix[ stype ] = {}
                    ready_matrix[ stype ][ sname ] = sobj
        
        for rtype in ready_matrix:
            log( "READY %s: %s" % ( rtype, ",".join(ready_matrix[rtype].keys()) ) )         

        for dType in ready_matrix:
            log("Found %s %d" % (dType, len(ready_matrix[dType])))
            
        merge_children = {}

        for merge_type in CGData.MERGE_OBJECTS:
            mtype = CGData.get_type( merge_type )
            select_types = mtype.typeSet
            select_set = {}
            try:
                for stype in select_types:
                    select_set[ stype ] = ready_matrix[ stype ] 
                    if stype not in merge_children:
                        merge_children[stype] = {}
            except KeyError:
                error("missing data type %s" % (stype) )
                continue
            mobjlist = self.set_enumerate( mtype, select_set )
            for mobj in mobjlist:
                if merge_type not in ready_matrix:
                    ready_matrix[ merge_type ] = {}
                for cType in mobj:
                    merge_children[cType][mobj[cType].get_name()] = True
                ready_matrix[ merge_type ][ mobj.get_name() ] = mobj
        
        self.compile_matrix = {}
        for sType in ready_matrix:
            self.compile_matrix[sType] = {}
            for name in ready_matrix[sType]:
                if sType not in merge_children or name not in merge_children[sType]:
                    self.compile_matrix[sType][name] = ready_matrix[sType][name]
       
        log("After Merge")
        for dType in ready_matrix:
            log("Found %s %d" % (dType, len(self.compile_matrix[dType])))
Пример #4
0
    def link_objects(self):
        """
        Scan found object records and determine if the data they link to is
        avalible
        """
        omatrix = {}
        for otype in self.set_hash:
            if issubclass(CGData.get_type(otype), CGData.CGGroupMember):
                gmap = {}
                for oname in self.set_hash[otype]:
                    oobj = self.set_hash[otype][oname]
                    if oobj.get_group() not in gmap:
                        gmap[oobj.get_group()] = CGData.CGGroupBase(
                            oobj.get_group())
                    gmap[oobj.get_group()].put(oobj)
                omatrix[otype] = gmap
            else:
                omatrix[otype] = self.set_hash[otype]

        # Now it's time to check objects for their dependencies
        ready_matrix = {}
        for stype in omatrix:
            for sname in omatrix[stype]:
                sobj = omatrix[stype][sname]
                lmap = sobj.get_link_map()
                is_ready = True
                for ltype in lmap:
                    if not omatrix.has_key(ltype):
                        warn("%s missing data type %s" % (sname, ltype))
                        is_ready = False
                    else:
                        for lname in lmap[ltype]:
                            if not omatrix[ltype].has_key(lname):
                                warn("%s %s missing data %s %s" %
                                     (stype, sname, ltype, lname))
                                is_ready = False
                if not sobj.is_link_ready():
                    warn("%s %s not LinkReady" % (stype, sname))
                elif is_ready:
                    if not stype in ready_matrix:
                        ready_matrix[stype] = {}
                    ready_matrix[stype][sname] = sobj

        for rtype in ready_matrix:
            log("READY %s: %s" % (rtype, ",".join(ready_matrix[rtype].keys())))

        for dType in ready_matrix:
            log("Found %s %d" % (dType, len(ready_matrix[dType])))

        merge_children = {}

        for merge_type in CGData.MERGE_OBJECTS:
            mtype = CGData.get_type(merge_type)
            select_types = mtype.typeSet
            select_set = {}
            try:
                for stype in select_types:
                    select_set[stype] = ready_matrix[stype]
                    if stype not in merge_children:
                        merge_children[stype] = {}
            except KeyError:
                error("missing data type %s" % (stype))
                continue
            mobjlist = self.set_enumerate(mtype, select_set)
            for mobj in mobjlist:
                if merge_type not in ready_matrix:
                    ready_matrix[merge_type] = {}
                for cType in mobj:
                    merge_children[cType][mobj[cType].get_name()] = True
                ready_matrix[merge_type][mobj.get_name()] = mobj

        self.compile_matrix = {}
        for sType in ready_matrix:
            self.compile_matrix[sType] = {}
            for name in ready_matrix[sType]:
                if sType not in merge_children or name not in merge_children[
                        sType]:
                    self.compile_matrix[sType][name] = ready_matrix[sType][
                        name]

        log("After Merge")
        for dType in ready_matrix:
            log("Found %s %d" % (dType, len(self.compile_matrix[dType])))