Esempio n. 1
0
def populate():
    if len(operations) == 0:
        add_operation(query.operation())
        add_operation(show_tables.operation())
        add_operation(delete_tag.operation())
        add_operation(report.operation())
        add_operation(run.operation())
        add_operation(show_db_config.operation())
        add_operation(graph_create.operation())
        add_operation(graph_v_ingest.operation())
        add_operation(graph_v_search.operation())
        add_operation(make_graph500.operation())
        add_operation(generate_elist.operation())
        add_operation(graph_e_ingest.operation())
        add_operation(graph_e_standard_ingest.operation())
        add_operation(graph_e_ring_ingest.operation())
        add_operation(graph_navigate_ring.operation())
        add_operation(graph_navigate_dense.operation())
        pass
    pass
Esempio n. 2
0
def populate():
    if len(operations) == 0:
        add_operation(query.operation())
        add_operation(show_tables.operation())
        add_operation(delete_tag.operation())
        add_operation(report.operation())
        add_operation(run.operation())
        add_operation(show_db_config.operation())
        add_operation(graph_create.operation())
        add_operation(graph_v_ingest.operation())
        add_operation(graph_v_search.operation())
        add_operation(make_graph500.operation())
        add_operation(generate_elist.operation())
        add_operation(graph_e_ingest.operation())
        add_operation(graph_e_standard_ingest.operation())
        add_operation(graph_e_ring_ingest.operation())
        add_operation(graph_navigate_ring.operation())
        add_operation(graph_navigate_dense.operation())
        pass
    pass
Esempio n. 3
0
    def run_operation(self):
        for engine in self.engine_objects:
            for _page_size in self.page_size:
                for _v_scale in self.scale:
                    for _e_factor in self.factor:
                        for _threads in self.threads:
                            for _txsize in self.txsize:
                                for _cache in self.cache:
                                    _index = "gr"
                                    _v_size = pow(2, _v_scale)
                                    self.initialize_property(engine.name)
                                    self.propertyFile.properties[
                                        "IG.Placement.Distributed.Pipelining.Groups"] = "ConnectorGroup"
                                    self.propertyFile.properties[
                                        "IG.Placement.Distributed.Pipelining.PipelinesPerStorageUnit"] = 4
                                    self.propertyFile.properties[
                                        "IG.Placement.Distributed.Pipelining.EnablePipelining"] = "true"
                                    self.propertyFile.setInitCache(_cache[0])
                                    self.propertyFile.setMaxCache(_cache[1])
                                    self.propertyFile.properties[
                                        "IG.PageSize"] = _page_size
                                    self.ig_setup_placement(
                                        engine.name, self.propertyFile)
                                    _s_ = "\tgraph(%s) create index:%s page_size:%d" % (
                                        engine.name, _index, _page_size)
                                    print self.output_string(
                                        _s_, base.Colors.Blue, True),
                                    sys.stdout.flush()
                                    start = time.clock()
                                    self.ig_run(engine.name, self.propertyFile,
                                                "create", _index)
                                    elapsed = (time.clock() - start)
                                    print self.output_string(
                                        str(elapsed), base.Colors.Red, False)
                                    self.ig_setup_Location(
                                        engine.name, self.propertyFile)

                                    _s_ = "\tgraph(%s) ingest edges index:%s page_size:%d tx_size:%d size:%d diskmap:%s" % (
                                        engine.name, _index, _page_size,
                                        _txsize, _v_size, str(self.diskmap))
                                    print self.output_string(
                                        _s_, base.Colors.Blue, True)
                                    vprofileName = "v_ingest.profile"
                                    eprofileName = "e_ingest.profile"
                                    elist_name = "g500.elist"
                                    now_string = self.db.now_string(
                                        True).replace(" ", "_")
                                    if self.tag_object:
                                        vprofileName = "v_ingest." + now_string + ".profile"
                                        eprofileName = "e_ingest." + now_string + ".profile"
                                        vprofileName = vprofileName.replace(
                                            " ", "_")
                                        eprofileName = eprofileName.replace(
                                            " ", "_")
                                        elist_name = "g500.%s.elist" % (
                                            now_string)
                                        pass
                                    self.ig_v_ingest(engine.name,
                                                     self.propertyFile, _index,
                                                     _v_size, 0, _threads,
                                                     _txsize, vprofileName,
                                                     True)
                                    generator = generate_elist.operation()
                                    generator.run(_v_scale, _e_factor, self.a,
                                                  self.b, self.c, self.d,
                                                  elist_name)
                                    self.ig_e_pipeline_ingest(
                                        engine.name, self.propertyFile,
                                        _v_scale, _threads, _txsize,
                                        eprofileName, elist_name)
                                    os.remove(elist_name)
                                    if self.case_object:
                                        f = file(eprofileName, "r")
                                        line = f.readline()
                                        data = eval(line)
                                        platform_object = self.db.create_unique_object(
                                            db_objects.model.platform, "name",
                                            data["os"])
                                        index_object = self.db.create_unique_object(
                                            db_objects.model.index_type,
                                            "name", _index)

                                        case_data_object = self.db.create_object(
                                            db_objects.model.case_data,
                                            timestamp=self.db.now_string(True),
                                            case_id=self.case_object.id,
                                            tag_id=self.tag_object.id,
                                            engine_id=engine.id,
                                            size=data["size"],
                                            time=data["time"],
                                            memory_init=data["mem_init"],
                                            memory_used=data["mem_used"],
                                            memory_committed=data[
                                                "mem_committed"],
                                            memory_max=data["mem_max"],
                                            op_size=data["opsize"],
                                            rate=data["rate"],
                                            page_size=_page_size,
                                            cache_init=self.propertyFile.
                                            getInitCache(),
                                            cache_max=self.propertyFile.
                                            getMaxCache(),
                                            tx_size=_txsize,
                                            platform_id=platform_object.id,
                                            threads=_threads,
                                            index_id=index_object.id,
                                            status=1)
                                        if self.diskmap:
                                            case_data_object.setDataValue(
                                                "diskmap", self.diskmap)
                                            pass
                                        case_data_object.setDataValue(
                                            "edge_factor", _e_factor)
                                        self.db.update(case_data_object)
                                        case_data_key = case_data_object.generateKey(
                                        )
                                        case_data_stat_object = self.db.fetch_using_generic(
                                            db_objects.model.case_data_stat,
                                            key=case_data_key,
                                            case_id=self.case_object.id)
                                        if (len(case_data_stat_object) == 0):
                                            case_data_stat_object = self.db.create_unique_object(
                                                db_objects.model.
                                                case_data_stat,
                                                "key",
                                                case_data_key,
                                                case_id=self.case_object.id)
                                        else:
                                            case_data_stat_object = case_data_stat_object[
                                                0]
                                            pass
                                        case_data_stat_object.addCounter()
                                        case_data_stat_object.setRateStat(
                                            data["rate"])
                                        case_data_stat_object.setTimeStat(
                                            data["time"])
                                        case_data_stat_object.setMemInitStat(
                                            data["mem_init"])
                                        case_data_stat_object.setMemUsedStat(
                                            data["mem_used"])
                                        case_data_stat_object.setMemCommittedStat(
                                            data["mem_committed"])
                                        case_data_stat_object.setMemMaxStat(
                                            data["mem_max"])
                                        self.db.update(case_data_stat_object)
                                        f.close()
                                        os.remove(eprofileName)
                                        os.remove(vprofileName)
                                        pass
                                    pass
                                pass
                            pass
                        pass
                    pass
                pass
            pass
        pass
Esempio n. 4
0
    def run_operation(self):
        for engine in self.engine_objects:
            for _page_size in self.page_size:
                for _v_scale in self.scale:
                    for _e_factor in self.factor:
                        for _threads in self.threads:
                            for _txsize in self.txsize:
                                for _cache in self.cache:
                                    _index = "gr"
                                    _v_size = pow(2,_v_scale)
                                    self.initialize_property(engine.name)
                                    self.propertyFile.properties["IG.Placement.Distributed.Pipelining.Groups"] = "ConnectorGroup"
                                    self.propertyFile.properties["IG.Placement.Distributed.Pipelining.PipelinesPerStorageUnit"]=4
                                    self.propertyFile.properties["IG.Placement.Distributed.Pipelining.EnablePipelining"]="true"
                                    self.propertyFile.setInitCache(_cache[0])
                                    self.propertyFile.setMaxCache(_cache[1])
                                    self.propertyFile.properties["IG.PageSize"] = _page_size
                                    self.ig_setup_placement(engine.name,self.propertyFile)
                                    _s_ = "\tgraph(%s) create index:%s page_size:%d"%(engine.name,_index,_page_size)
                                    print self.output_string(_s_,base.Colors.Blue,True),
                                    sys.stdout.flush()
                                    start = time.clock()
                                    self.ig_run(engine.name,self.propertyFile,"create",_index)
                                    elapsed = (time.clock() - start)
                                    print self.output_string(str(elapsed),base.Colors.Red,False)
                                    self.ig_setup_Location(engine.name,self.propertyFile)
                                

                                    _s_ = "\tgraph(%s) ingest edges index:%s page_size:%d tx_size:%d size:%d diskmap:%s"%(engine.name,_index,_page_size,_txsize,_v_size,str(self.diskmap))
                                    print self.output_string(_s_,base.Colors.Blue,True)
                                    vprofileName = "v_ingest.profile"
                                    eprofileName = "e_ingest.profile"
                                    elist_name  = "g500.elist"
                                    now_string = self.db.now_string(True).replace(" ","_")
                                    if self.tag_object:
                                        vprofileName = "v_ingest."+ now_string  +  ".profile"
                                        eprofileName = "e_ingest."+ now_string +  ".profile"
                                        vprofileName = vprofileName.replace(" ","_")
                                        eprofileName = eprofileName.replace(" ","_")
                                        elist_name = "g500.%s.elist"%(now_string)
                                        pass
                                    self.ig_v_ingest(engine.name,self.propertyFile,_index,_v_size,0,_threads,_txsize,vprofileName,True)
                                    generator = generate_elist.operation()
                                    generator.run(_v_scale,_e_factor,self.a,self.b,self.c,self.d,elist_name)
                                    self.ig_e_pipeline_ingest(engine.name,self.propertyFile,_v_scale,_threads,_txsize,eprofileName,elist_name)
                                    os.remove(elist_name)
                                    if self.case_object:
                                        f = file(eprofileName,"r")
                                        line = f.readline()
                                        data = eval(line)
                                        platform_object = self.db.create_unique_object(db_objects.model.platform,"name",data["os"])
                                        index_object = self.db.create_unique_object(db_objects.model.index_type,"name",_index)
                                    
                                        case_data_object = self.db.create_object(db_objects.model.case_data,
                                                                                 timestamp=self.db.now_string(True),
                                                                                 case_id=self.case_object.id,
                                                                                 tag_id=self.tag_object.id,
                                                                                 engine_id=engine.id,
                                                                                 size=data["size"],
                                                                                 time=data["time"],
                                                                                 memory_init=data["mem_init"],
                                                                                 memory_used=data["mem_used"],
                                                                                 memory_committed=data["mem_committed"],
                                                                                 memory_max=data["mem_max"],
                                                                                 op_size=data["opsize"],
                                                                                 rate=data["rate"],
                                                                                 page_size=_page_size,
                                                                                 cache_init=self.propertyFile.getInitCache(),
                                                                                 cache_max=self.propertyFile.getMaxCache(),
                                                                                 tx_size=_txsize,
                                                                                 platform_id=platform_object.id,
                                                                                 threads=_threads,
                                                                                 index_id=index_object.id,
                                                                                 status=1
                                                                                 )
                                        if self.diskmap:
                                            case_data_object.setDataValue("diskmap",self.diskmap)
                                            pass
                                        case_data_object.setDataValue("edge_factor",_e_factor)
                                        self.db.update(case_data_object)
                                        case_data_key = case_data_object.generateKey()
                                        case_data_stat_object = self.db.fetch_using_generic(db_objects.model.case_data_stat,
                                                                                            key=case_data_key,
                                                                                            case_id=self.case_object.id
                                                                                            )
                                        if (len(case_data_stat_object) == 0):
                                            case_data_stat_object = self.db.create_unique_object(db_objects.model.case_data_stat,
                                                                                                 "key",case_data_key,
                                                                                                 case_id=self.case_object.id
                                                                                                 )
                                        else:
                                            case_data_stat_object = case_data_stat_object[0]
                                            pass
                                        case_data_stat_object.addCounter()
                                        case_data_stat_object.setRateStat(data["rate"])
                                        case_data_stat_object.setTimeStat(data["time"])
                                        case_data_stat_object.setMemInitStat(data["mem_init"])
                                        case_data_stat_object.setMemUsedStat(data["mem_used"])
                                        case_data_stat_object.setMemCommittedStat(data["mem_committed"])
                                        case_data_stat_object.setMemMaxStat(data["mem_max"])
                                        self.db.update(case_data_stat_object)
                                        f.close()
                                        os.remove(eprofileName)
                                        os.remove(vprofileName)
                                        pass
                                    pass
                                pass
                            pass
                        pass
                    pass
                pass
            pass
        pass