Exemplo n.º 1
0
    def initialize(self, argv):
        parser = optparse.OptionParser(
            usage="%prog <target-system> [options] <dgen-config>",
            version=self.VERSION)
        self.configureParser(parser)
        self.parser = parser

        args, remainder = parser.parse_args(argv)
        if (len(remainder) != 1):
            self.error(None, True)
            raise

        self.log = sysutil.createLogger("myriad.importer")

        try:
            # process command line arguments
            self.setup(args, remainder)

            if (self.logBase == None):
                # create log dir
                self.logBase = tempfile.mkdtemp(
                    "", "myriad-import-%s_" % (self.datasetID))

            # make sure that logBase directories exist
            sysutil.checkDir(self.logBase)

            # register file handler to the logger
            sysutil.registerFileHandler(
                self.log, "%s/myriad-importer.log" % (self.logBase))

        except config.UnknownConfigObjectException, e:
            self.error(str(e), True)
            raise e
Exemplo n.º 2
0
    def initialize(self, argv):
        
        parser = optparse.OptionParser(usage="%prog [options] <dgen-config>", version=self.VERSION)
        parser.add_option("-s", dest="sf", type="float", default=1.0,
                          help="scaling factor (sf=1 generates 1GB data)")
        parser.add_option("-m", dest="dataset_id", type="str", default="default-dataset",
                          help="ID of the generated Myriad dataset")
        parser.add_option("-x", dest="execute_stages", action="append", type="str", default=[],
                          help="Specify a specific stage to execute")
        parser.add_option("-n", dest="node_config", type="str", default="%s-node.xml" % (self.dgenName),
                          help="name of the node config file (should reside in the config dir)")
        parser.add_option("--config-dir", dest="config_path", type="str", default="%s/config" % (self.basePath),
                          help="path to the myriad config folder (TODO)")
        parser.add_option("--log-dir", dest="log_dir", type="str", default=None, 
                          help="base directory for output logging")
        parser.add_option("--cleanup", dest="cleanup", action="store_true",
                          help="remove output from previously generated job")
            
        self.parser = parser

        args, remainder = parser.parse_args(argv)
        if (len(remainder) != 1):
            self.error(None, True)
            raise
        
        self.log = sysutil.createLogger("myriad.dgen")

        try:
            self.cleanup = args.cleanup
            self.sf = args.sf
            self.datasetID = args.dataset_id
            self.configPath = args.config_path
            self.nodeConfig = args.node_config
            self.logBase = args.log_dir
            self.executeStages = args.execute_stages
            self.configName = remainder.pop()
            
            # load myriad config 
            self.config = config.readConfig(self.dgenName, self.nodeConfig, "%s/%s-frontend.xml" % (self.configPath, self.dgenName))
            # load sizing config
            self.dgenConfig = config.readDGenConfig("%s/%s-node.properties" % (self.configPath, self.dgenName))
            
            DGenNode.MAX_ATTEMPTS = int(self.dgenConfig.getProperty("coordinator.node.max.attempts", DGenNode.MAX_ATTEMPTS))
            DGenNode.DEAD_TIMEOUT = datetime.timedelta(0, 0, 0, int(self.dgenConfig.getProperty("coordinator.node.dead.timeout", DGenNode.DEAD_TIMEOUT.seconds*1000)))
            NodeMonitor.POLL_INTERVAL = int(self.dgenConfig.getProperty("coordinator.node.monitor.interval", NodeMonitor.POLL_INTERVAL*1000))/1000.0
            
            if (self.logBase == None):
                # create log dir
                self.logBase = tempfile.mkdtemp("", "%s-frontend-%s_" % (self.dgenName, self.datasetID))
            
            # make sure that logBase directories exist
            sysutil.checkDir(self.logBase)
            
            # register file handler to the logger
            sysutil.registerFileHandler(self.log, "%s/%s-frontend.log" % (self.logBase, self.dgenName))

        except:
            e = sys.exc_info()[1]
            self.error("unexpected error: %s" % (str(e)), True)
            raise
Exemplo n.º 3
0
    def initialize(self, argv):
        parser = optparse.OptionParser(usage="%prog <target-system> [options] <dgen-config>", version=self.VERSION)
        self.configureParser(parser)
        self.parser = parser

        args, remainder = parser.parse_args(argv)
        if (len(remainder) != 1):
            self.error(None, True)
            raise
        
        self.log = sysutil.createLogger("myriad.importer")
        
        try:
            # process command line arguments
            self.setup(args, remainder)
                    
            if (self.logBase == None):
                # create log dir
                self.logBase = tempfile.mkdtemp("", "myriad-import-%s_" % (self.datasetID))
            
            # make sure that logBase directories exist
            sysutil.checkDir(self.logBase)
            
            # register file handler to the logger
            sysutil.registerFileHandler(self.log, "%s/myriad-importer.log" % (self.logBase))
            
        except config.UnknownConfigObjectException, e:
            self.error(str(e), True)
            raise e