Example #1
0
    def generateTaskList(self, xVars, paraCombos, substs, sessionParas):
        pathToExecutable = sessionParas['pathToExecutable']
        localBaseDir = sessionParas['localBaseDir']
        architecture = sessionParas['architecture']
        jobname = sessionParas['jobname']
        # Fill the task list
        tasks = []
        for paraCombo in paraCombos:
            executable = os.path.basename(pathToExecutable)
            inputs = {pathToExecutable: executable}
            # make a "stage" directory where input files are collected
            path_to_stage_dir = os.path.join(self.optimFolder, jobname)

            # 2. apply substitutions to parameter files in local base dir
            for (path, changes) in substs.iteritems():
                for (var, val, index, regex) in changes:
                    support.update_parameter_in_file(
                        os.path.join(localBaseDir, path), var, index, val,
                        regex)
                # adjust xVar in parameter file
                index = 0
                regex = 'bar-separated'
                for xVar, xVal in zip(xVars, paraCombo):
                    support.update_parameter_in_file(
                        os.path.join(localBaseDir, path), xVar, 0, xVal, regex)
                    path_to_stage_dir += '_' + xVar + '=' + str(xVal)
            gc3libs.utils.mkdir(path_to_stage_dir)
            prefix_len = len(path_to_stage_dir) + 1
            # fill stage dir
            self.fillInputDir(localBaseDir, path_to_stage_dir)
            # 3. build input file list
            for dirpath, dirnames, filenames in os.walk(path_to_stage_dir):
                for filename in filenames:
                    # cut the leading part, which is == to path_to_stage_dir
                    relpath = dirpath[prefix_len:]
                    # ignore output directory contents in resubmission
                    if relpath.startswith('output'):
                        continue
                    remote_path = os.path.join(relpath, filename)
                    inputs[os.path.join(dirpath, filename)] = remote_path
            # all contents of the `output` directory are to be fetched
            outputs = {'output/': ''}
            #kwargs = sessionParas.copy()
            kwargs = {}
            kwargs['jobname'] = self.jobname
            ##            kwargs.pop('requested_memory')
            ##            kwargs.pop('requested_walltime')
            ##            kwargs.pop('requested_cores')
            kwargs['stdout'] = 'forwardPremiumOut.log'
            kwargs['join'] = True
            kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output')
            kwargs['requested_architecture'] = architecture

            # hand over job to create
            tasks.append(
                forwardPremium.GPremiumApplication('./' + executable, [],
                                                   inputs, outputs, **kwargs))
        return tasks
Example #2
0
    def generateTaskList(self, xVars, paraCombos, substs, sessionParas):
        pathToExecutable = sessionParas['pathToExecutable']
        localBaseDir     = sessionParas['localBaseDir']
        architecture     = sessionParas['architecture']
        jobname          = sessionParas['jobname']
        # Fill the task list
        tasks = []
        for paraCombo in paraCombos:
            executable = os.path.basename(pathToExecutable)
            inputs = { pathToExecutable:executable }        
            # make a "stage" directory where input files are collected
            path_to_stage_dir = os.path.join(self.optimFolder, jobname)
            
            # 2. apply substitutions to parameter files in local base dir
            for (path, changes) in substs.iteritems():
                for (var, val, index, regex) in changes:
                    support.update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex)
                # adjust xVar in parameter file
                index = 0
                regex = 'bar-separated'
                for xVar, xVal in zip(xVars, paraCombo):
                    support.update_parameter_in_file(os.path.join(localBaseDir, path), xVar, 0, xVal, regex)
                    path_to_stage_dir += '_' + xVar + '=' + str(xVal)
            gc3libs.utils.mkdir(path_to_stage_dir)
            prefix_len = len(path_to_stage_dir) + 1 
            # fill stage dir
            self.fillInputDir(localBaseDir, path_to_stage_dir)
            # 3. build input file list
            for dirpath,dirnames,filenames in os.walk(path_to_stage_dir):
                for filename in filenames:
                    # cut the leading part, which is == to path_to_stage_dir
                    relpath = dirpath[prefix_len:]
                    # ignore output directory contents in resubmission
                    if relpath.startswith('output'):
                        continue
                    remote_path = os.path.join(relpath, filename)
                    inputs[os.path.join(dirpath, filename)] = remote_path
            # all contents of the `output` directory are to be fetched
            outputs = { 'output/':'' }
            #kwargs = sessionParas.copy()
            kwargs = {}
            kwargs['jobname'] = self.jobname
##            kwargs.pop('requested_memory')
##            kwargs.pop('requested_walltime')
##            kwargs.pop('requested_cores')
            kwargs['stdout'] = 'forwardPremiumOut.log'
            kwargs['join'] = True
            kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output')
            kwargs['requested_architecture'] = architecture

            # hand over job to create
            tasks.append(forwardPremium.GPremiumApplication('./' + executable, [], inputs, outputs, **kwargs)) 
        return tasks
    def generateTaskList(self, xVars, paraCombos, substs, sessionParas):
        # setup AppPot parameters
        use_apppot = False
        apppot_img = None
        apppot_changes = None
        apppot_file = sessionParas['AppPotFile']
        if apppot_file:
            # print 'apppot_file = %s' % apppot_file
            # os._exit(1)
            use_apppot = True
            if apppot_file.endswith('.changes.tar.gz'):
                apppot_changes = apppot_file
            else:
                apppot_img = apppot_file

        pathToExecutable = sessionParas['pathToExecutable']
        localBaseDir = sessionParas['localBaseDir']
        architecture = sessionParas['architecture']
        jobname = sessionParas['jobname']
        rte = sessionParas['rte']
        # Fill the task list
        tasks = []
        for paraCombo in paraCombos:
            logger.debug('paraCombo = %s' % paraCombo)
            executable = os.path.basename(pathToExecutable)
            inputs = {pathToExecutable: executable}
            # make a "stage" directory where input files are collected
            path_to_stage_dir = os.path.join(self.optimFolder, jobname)

            # 2. apply substitutions to parameter files in local base dir
            for (path, changes) in substs.iteritems():
                for (var, val, index, regex) in changes:
                    support.update_parameter_in_file(
                        os.path.join(localBaseDir, path), var, index, val,
                        regex)
                # adjust xVar in parameter file
                index = 0
                regex = 'bar-separated'
                for xVar, xVal in zip(xVars, paraCombo):
                    support.update_parameter_in_file(
                        os.path.join(localBaseDir, path), xVar, 0, xVal, regex)
                    path_to_stage_dir += '_' + xVar + '=' + str(xVal)
            gc3libs.utils.mkdir(path_to_stage_dir)
            prefix_len = len(path_to_stage_dir) + 1
            # fill stage dir
            fillInputDir(localBaseDir, path_to_stage_dir)
            # 3. build input file list
            for dirpath, dirnames, filenames in os.walk(path_to_stage_dir):
                for filename in filenames:
                    # cut the leading part, which is == to path_to_stage_dir
                    relpath = dirpath[prefix_len:]
                    # ignore output directory contents in resubmission
                    if relpath.startswith('output'):
                        continue
                    remote_path = os.path.join(relpath, filename)
                    inputs[os.path.join(dirpath, filename)] = remote_path
            # all contents of the `output` directory are to be fetched
            outputs = {'output/': ''}
            #kwargs = sessionParas.copy()
            kwargs = {}
            kwargs['jobname'] = self.jobname
            kwargs['stdout'] = 'idriskOut.log'
            kwargs['join'] = True
            kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output')
            kwargs['requested_architecture'] = architecture

            # adaptions for uml
            if use_apppot:
                if apppot_img is not None:
                    kwargs['apppot_img'] = apppot_img
                if apppot_changes is not None:
                    kwargs['apppot_changes'] = apppot_changes
                cls = idRiskApppotApplication
                callExecutable = '/home/user/job/' + executable
            elif rte:
                kwargs['apppot_tag'] = 'ENV/APPPOT-0.26'
                kwargs['tags'] = ['TEST/APPPOT-IBF-1.0']
                cls = idRiskApppotApplication
                callExecutable = '/home/user/job/' + executable
            else:
                cls = idRiskApplication
                callExecutable = './' + executable
            kwargs.setdefault('tags', [])
            print 'cls = %s' % cls
            print 'callExecutable = %s' % callExecutable
            print 'kwargs = %s' % kwargs

            # hand over job to create
            curApplication = cls(callExecutable, [], inputs, outputs, **kwargs)
            tasks.append(curApplication)
        return tasks
Example #4
0
    def __init__(self, paraFolder, substs, solverParas, **sessionParas):

        logger.debug('entering solveParaCombination.__init__ for job %s' % sessionParas['jobname'])
        self.iter    = 0

        self.jobname = 'idRiskParaSearchDriver' + sessionParas['jobname']
        self.substs = substs

        self.sessionParas     = sessionParas
        self.pathToExecutable = sessionParas['pathToExecutable']
        self.architecture     = sessionParas['architecture']
        self.localBaseDir     = sessionParas['localBaseDir']
        
        self.paraFolder = os.path.join(os.getcwd(), sessionParas['jobname'])
        # setup AppPot parameters
        use_apppot = False
        apppot_img = None  
        apppot_changes = None
        apppot_file = sessionParas['AppPotFile']
        if apppot_file:
            use_apppot = True
            if apppot_file.endswith('.changes.tar.gz'):
                apppot_changes = apppot_file
            else:
                apppot_img = apppot_file
        pathToExecutable = sessionParas['pathToExecutable']
        localBaseDir     = sessionParas['localBaseDir']
        architecture     = sessionParas['architecture']
        jobname          = sessionParas['jobname']
        executable = os.path.basename(self.pathToExecutable)
        inputs = { self.pathToExecutable:executable }
        # make a "stage" directory where input files are collected
        path_to_stage_dir = os.path.join(self.paraFolder, jobname)
        gc3libs.utils.mkdir(path_to_stage_dir)
        prefix_len = len(path_to_stage_dir) + 1
        # 2. apply substitutions to parameter files
        for (path, changes) in substs.iteritems():
            for (var, val, index, regex) in changes:
                support.update_parameter_in_file(os.path.join(localBaseDir, path),
                                         var, index, val, regex)
        support.fillInputDir(localBaseDir, path_to_stage_dir)
        # 3. build input file list
        for dirpath,dirnames,filenames in os.walk(path_to_stage_dir):
            for filename in filenames:
                # cut the leading part, which is == to path_to_stage_dir
                relpath = dirpath[prefix_len:]
                # ignore output directory contents in resubmission
                if relpath. startswith('output'):
                    continue
                remote_path = os.path.join(relpath, filename)
                inputs[os.path.join(dirpath, filename)] = remote_path
        # all contents of the `output` directory are to be fetched
        outputs = { 'output/':'' }
        kwargs = {}
        kwargs['stdout'] = 'idRisk.log'
        kwargs['join'] = True
        kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output')
        kwargs['requested_architecture'] = self.architecture
        
        # adaptions for uml
        if use_apppot:
            if apppot_img is not None:
                kwargs['apppot_img'] = apppot_img
            if apppot_changes is not None:
                kwargs['apppot_changes'] = apppot_changes
            cls = idRiskApppotApplication
        else:
            cls = idRiskApplication 
        kwargs.setdefault('tags', [ ])

        # hand over job to create
#        self.curApplication = cls('/home/user/job/' + executable, [], inputs, outputs, **kwargs)
        self.curApplication = cls('./' + executable, [], inputs, outputs, **kwargs)

        SequentialTaskCollection.__init__(self, self.jobname, [ self.curApplication ])

        logger.debug('done gParaSearchDriver.__init__ for job %s' % sessionParas['jobname'])
Example #5
0
    def generateTaskList(self, xVars, paraCombos, substs, sessionParas):
        # setup AppPot parameters
        use_apppot = False
        apppot_img = None
        apppot_changes = None
        apppot_file = sessionParas["AppPotFile"]
        if apppot_file:
            use_apppot = True
            if apppot_file.endswith(".changes.tar.gz"):
                apppot_changes = apppot_file
            else:
                apppot_img = apppot_file

        pathToExecutable = sessionParas["pathToExecutable"]
        localBaseDir = sessionParas["localBaseDir"]
        architecture = sessionParas["architecture"]
        jobname = sessionParas["jobname"]
        # Fill the task list
        tasks = []
        for paraCombo in paraCombos:
            logger.debug("paraCombo = %s" % paraCombo)
            executable = os.path.basename(pathToExecutable)
            inputs = {pathToExecutable: executable}
            # make a "stage" directory where input files are collected
            path_to_stage_dir = os.path.join(self.optimFolder, jobname)

            # 2. apply substitutions to parameter files in local base dir
            for (path, changes) in substs.iteritems():
                for (var, val, index, regex) in changes:
                    support.update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex)
                # adjust xVar in parameter file
                index = 0
                regex = "bar-separated"
                for xVar, xVal in zip(xVars, paraCombo):
                    support.update_parameter_in_file(os.path.join(localBaseDir, path), xVar, 0, xVal, regex)
                    path_to_stage_dir += "_" + xVar + "=" + str(xVal)
            gc3libs.utils.mkdir(path_to_stage_dir)
            prefix_len = len(path_to_stage_dir) + 1
            # fill stage dir
            fillInputDir(localBaseDir, path_to_stage_dir)
            # 3. build input file list
            for dirpath, dirnames, filenames in os.walk(path_to_stage_dir):
                for filename in filenames:
                    # cut the leading part, which is == to path_to_stage_dir
                    relpath = dirpath[prefix_len:]
                    # ignore output directory contents in resubmission
                    if relpath.startswith("output"):
                        continue
                    remote_path = os.path.join(relpath, filename)
                    inputs[os.path.join(dirpath, filename)] = remote_path
            # all contents of the `output` directory are to be fetched
            outputs = {"output/": ""}
            # kwargs = sessionParas.copy()
            kwargs = {}
            kwargs["jobname"] = self.jobname
            kwargs["stdout"] = "idriskOut.log"
            kwargs["join"] = True
            kwargs["output_dir"] = os.path.join(path_to_stage_dir, "output")
            kwargs["requested_architecture"] = architecture

            # adaptions for uml
            if use_apppot:
                if apppot_img is not None:
                    kwargs["apppot_img"] = apppot_img
                if apppot_changes is not None:
                    kwargs["apppot_changes"] = apppot_changes
                cls = idRiskApppotApplication
            else:
                cls = idRiskApplication
            kwargs.setdefault("tags", [])

            # hand over job to create
            curApplication = cls("./" + executable, [], inputs, outputs, **kwargs)
            tasks.append(curApplication)
        return tasks
    def generateTaskList(self, xVars, paraCombos, substs, sessionParas):
        # setup AppPot parameters
        use_apppot = False
        apppot_img = None
        apppot_changes = None
        apppot_file = sessionParas['AppPotFile']
        if apppot_file:
            # print 'apppot_file = %s' % apppot_file
            # os._exit(1)
            use_apppot = True
            if apppot_file.endswith('.changes.tar.gz'):
                apppot_changes = apppot_file
            else:
                apppot_img = apppot_file

        pathToExecutable = sessionParas['pathToExecutable']
        localBaseDir     = sessionParas['localBaseDir']
        architecture     = sessionParas['architecture']
        jobname          = sessionParas['jobname']
        rte              = sessionParas['rte']
        # Fill the task list
        tasks = []
        for paraCombo in paraCombos:
            logger.debug('paraCombo = %s' % paraCombo)
            executable = os.path.basename(pathToExecutable)
            inputs = { pathToExecutable:executable }
            # make a "stage" directory where input files are collected
            path_to_stage_dir = os.path.join(self.optimFolder, jobname)

            # 2. apply substitutions to parameter files in local base dir
            for (path, changes) in substs.iteritems():
                for (var, val, index, regex) in changes:
                    support.update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex)
                # adjust xVar in parameter file
                index = 0
                regex = 'bar-separated'
                for xVar, xVal in zip(xVars, paraCombo):
                    support.update_parameter_in_file(os.path.join(localBaseDir, path), xVar, 0, xVal, regex)
                    path_to_stage_dir += '_' + xVar + '=' + str(xVal)
            gc3libs.utils.mkdir(path_to_stage_dir)
            prefix_len = len(path_to_stage_dir) + 1
            # fill stage dir
            fillInputDir(localBaseDir, path_to_stage_dir)
            # 3. build input file list
            for dirpath,dirnames,filenames in os.walk(path_to_stage_dir):
                for filename in filenames:
                    # cut the leading part, which is == to path_to_stage_dir
                    relpath = dirpath[prefix_len:]
                    # ignore output directory contents in resubmission
                    if relpath.startswith('output'):
                        continue
                    remote_path = os.path.join(relpath, filename)
                    inputs[os.path.join(dirpath, filename)] = remote_path
            # all contents of the `output` directory are to be fetched
            outputs = { 'output/':'' }
            #kwargs = sessionParas.copy()
            kwargs = {}
            kwargs['jobname'] = self.jobname
            kwargs['stdout'] = 'idriskOut.log'
            kwargs['join'] = True
            kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output')
            kwargs['requested_architecture'] = architecture

            # adaptions for uml
            if use_apppot:
                if apppot_img is not None:
                    kwargs['apppot_img'] = apppot_img
                if apppot_changes is not None:
                    kwargs['apppot_changes'] = apppot_changes
                cls = idRiskApppotApplication
                callExecutable = '/home/user/job/' + executable
            elif rte:
                kwargs['apppot_tag'] = 'ENV/APPPOT-0.26'
                kwargs['tags'] = ['TEST/APPPOT-IBF-1.0']
                cls = idRiskApppotApplication
                callExecutable = '/home/user/job/' + executable
            else:
                cls = idRiskApplication
                callExecutable = './' + executable
            kwargs.setdefault('tags', [ ])
            print 'cls = %s' % cls
            print 'callExecutable = %s' % callExecutable
            print 'kwargs = %s' % kwargs

            # hand over job to create
            curApplication = cls(callExecutable, [], inputs, outputs, **kwargs)
            tasks.append(curApplication)
        return tasks