def __init__(self, videofile, email_from, smtp_server, **extra): self.videofile = videofile videofilename = os.path.basename(videofile) inboxdir = os.path.dirname(videofile) self.vdescrfile = os.path.join(inboxdir, 'video.description.txt') self.inboxdir = inboxdir self.email_to = '' self.email_from = email_from self.smtp_server = smtp_server self.video_is_needed = True # Look for a 'configuration' file in the inboxdir to override # bemovi parameters videoname = videofilename.rsplit('.', 1)[0] self.extra = extra extra['jobname'] = 'BemoviWorkflow_%s' % videoname extra['videoname'] = videoname extra['base_output_dir'] = os.path.join( inboxdir + '.out', extra['jobname']) self.update_configuration() plocator = ParticleLocator(videofile, **extra) plinker = ParticleLinker( os.path.join(extra['base_output_dir'], ParticleLocator.application, extra['videoname'] + '.ijout.txt',), **extra) SequentialTaskCollection.__init__(self, [plocator, plinker], **extra)
def __init__(self, grayscaled_image, output_dir, output_file, colors, warhol_dir, **extra_args): self.grayscaled_image = grayscaled_image self.output_dir = output_dir self.warhol_dir = warhol_dir self.jobname = 'TricolorizeImage' self.output_file = output_file if not os.path.isdir(output_dir): os.mkdir(output_dir) self.extra_args = extra_args gc3libs.log.info( "Tricolorize image %s to %s" % ( self.grayscaled_image, self.output_file)) self.tasks = [ CreateLutApplication( self.grayscaled_image, "%s.miff" % self.grayscaled_image, self.output_dir, colors, self.warhol_dir, **extra_args), ] SequentialTaskCollection.__init__(self, self.tasks)
def __init__(self, jobname='', path_to_stage_dir='', opt_algorithm=None, task_constructor=None, extract_value_fn=(lambda app: app.value), cur_pop_file='', **extra_args): gc3libs.log.debug('entering ParallelDriver.__init__') # Set up initial variables and set the correct methods. self.jobname = jobname self.path_to_stage_dir = path_to_stage_dir self.opt_algorithm = opt_algorithm self.extract_value_fn = extract_value_fn self.task_constructor = task_constructor self.cur_pop_file = cur_pop_file self.extra_args = extra_args self.new_pop = self.opt_algorithm.pop initial_task = ComputeTargetVals(self.opt_algorithm.pop, self.jobname, self.opt_algorithm.cur_iter, path_to_stage_dir, self.cur_pop_file, task_constructor) SequentialTaskCollection.__init__(self, [initial_task], **extra_args)
def __init__(self, substs, solverParas, **sessionParas): logger.debug('entering solveParaCombination.__init__ for job %s' % sessionParas['jobname']) self.iter = 0 self.jobname = sessionParas['jobname'] self.substs = substs self.sessionParas = sessionParas self.pathToExecutable = sessionParas['pathToExecutable'] self.architecture = sessionParas['architecture'] self.localBaseDir = sessionParas['localBaseDir'] self.paraFolder = os.path.join(os.getcwd(), sessionParas['jobname']) self.wBarLower_task = idRiskParaSearchDriver(self.paraFolder, self.substs, solverParas, **sessionParas) SequentialTaskCollection.__init__(self, self.jobname, [self.wBarLower_task]) logger.debug('done gParaSearchDriver.__init__ for job %s' % sessionParas['jobname'])
def __init__(self, init_value, add_value, iterations, **extra_args): """ Create a new tasks that runs `executable` over a set of values (initially given by `initial_values_file`, then the output of a run is fed as input to the next one), riterating the process `total_iterations` times. If `slice_size` is a positive integer, then chop the input into chunks of -at most- the given size and compute them as separate independent jobs. Extra keyword arguments are saved and passed down to construct the `ValueFunctionIterationApplication`. """ self.init = init_value self.increment = add_value self.limit = iterations self.jobname = "Gdemo_Iteration" gc3libs.log.info("Calling DemoIteration.__init__() ... ") # create initial task and register it initial_task = GdemoApplication(self.init, self.increment, 0) SequentialTaskCollection.__init__(self, [initial_task], **extra_args)
def __init__(self, paraFolder, substs, solverParas, **sessionParas): logger.debug('entering gParaSearchDriver.__init__') self.jobname = sessionParas['jobname'] + 'driver' solverParas['jobname'] = self.jobname self.sessionParas = sessionParas self.solverParas = solverParas self.substs = substs self.paraFolder = paraFolder self.iter = 0 # create a subfolder for optim over xVar self.optimFolder = os.path.join(paraFolder, 'optim' + str(solverParas['xVars'][0])) self.solverParas['optimFolder'] = self.optimFolder gc3libs.utils.mkdir(self.optimFolder) self.costlyOptimizer = costlyOptimization.costlyOptimization(solverParas) self.xVars = solverParas['xVars'] self.xParaCombos = solverParas['xInitialParaCombo'] self.targetVars = solverParas['targetVars'] self.target_fx = solverParas['target_fx'] self.evaluator = idRiskParaSearchParallel(self.xVars, self.xParaCombos, substs, self.optimFolder, solverParas , **sessionParas) initial_task = self.evaluator SequentialTaskCollection.__init__(self, self.jobname, [initial_task]) logger.debug('done solveParaCombination.__init__')
def __init__(self, paraFolder, substs, solverParas, **sessionParas): logger.debug('entering gParaSearchDriver.__init__') self.jobname = sessionParas['jobname'] + 'driver' solverParas['jobname'] = self.jobname self.sessionParas = sessionParas self.solverParas = solverParas self.substs = substs self.paraFolder = paraFolder self.iter = 0 # create a subfolder for optim over xVar self.optimFolder = os.path.join(paraFolder, 'optim' + str(solverParas['xVars'][0])) gc3libs.utils.mkdir(self.optimFolder) self.costlyOptimizer = costlyOptimization.costlyOptimization(solverParas) self.xVars = solverParas['xVars'] self.xParaCombos = solverParas['xInitialParaCombo'] self.targetVars = solverParas['targetVars'] self.target_fx = solverParas['target_fx'] self.evaluator = idRiskParaSearchParallel(self.xVars, self.xParaCombos, substs, self.optimFolder, solverParas , **sessionParas) initial_task = self.evaluator SequentialTaskCollection.__init__(self, self.jobname, [initial_task]) logger.debug('done solveParaCombination.__init__')
def __init__(self, name, experiment_id, verbosity, submission_id, user_name, parent_id, description=None, waiting_time=0): ''' Parameters ---------- name: str name of the stage experiment_id: int ID of the processed experiment verbosity: int logging verbosity index submission_id: int ID of the corresponding submission user_name: str name of the submitting user parent_id: int ID of the parent :class:`Workflow <tmlib.workflow.workflow.Workflow>` description: tmlib.tmaps.description.WorkflowStageDescription, optional description of the stage (default: ``None``) waiting_time: int, optional time in seconds that should be waited upon transition from one stage to the other to avoid issues related to network file systems (default: ``0``) ''' SequentialTaskCollection.__init__( self, tasks=None, jobname='%s' % name ) WorkflowStage.__init__( self, name=name, experiment_id=experiment_id, verbosity=verbosity, submission_id=submission_id, description=description, user_name=user_name, parent_id=parent_id ) self.waiting_time = waiting_time
def __init__(self, paraFolder, substs, solverParas, **sessionParas): logger.debug("entering gParaSearchDriver.__init__") self.jobname = sessionParas["jobname"] + "driver" solverParas["jobname"] = self.jobname self.sessionParas = sessionParas self.solverParas = solverParas self.substs = substs self.paraFolder = paraFolder self.iter = 0 # create a subfolder for optim over xVar self.optimFolder = os.path.join(paraFolder, "optim" + str(solverParas["xVars"][0])) self.solverParas["optimFolder"] = self.optimFolder gc3libs.utils.mkdir(self.optimFolder) self.costlyOptimizer = costlyOptimization.costlyOptimization(solverParas) self.xVars = solverParas["xVars"] self.xParaCombos = solverParas["xInitialParaCombo"] self.targetVars = solverParas["targetVars"] self.target_fx = solverParas["target_fx"] self.evaluator = idRiskParaSearchParallel( self.xVars, self.xParaCombos, substs, self.optimFolder, solverParas, **sessionParas ) initial_task = self.evaluator SequentialTaskCollection.__init__(self, self.jobname, [initial_task]) logger.debug("done solveParaCombination.__init__")
def __init__(self, videofile, email_from, smtp_server, **extra): self.videofile = videofile videofilename = os.path.basename(videofile) inboxdir = os.path.dirname(videofile) self.vdescrfile = os.path.join(inboxdir, 'video.description.txt') self.inboxdir = inboxdir self.email_to = '' self.email_from = email_from self.smtp_server = smtp_server self.video_is_needed = True # Look for a 'configuration' file in the inboxdir to override # bemovi parameters videoname = videofilename.rsplit('.', 1)[0] self.extra = extra extra['jobname'] = 'BemoviWorkflow_%s' % videoname extra['videoname'] = videoname extra['base_output_dir'] = os.path.join(inboxdir + '.out', extra['jobname']) self.update_configuration() plocator = ParticleLocator(videofile, **extra) plinker = ParticleLinker( os.path.join( extra['base_output_dir'], ParticleLocator.application, extra['videoname'] + '.ijout.txt', ), **extra) SequentialTaskCollection.__init__(self, [plocator, plinker], **extra)
def __init__(self, executable, initial_values_file, total_iterations, slice_size=0, output_dir=TMPDIR, **extra_args): """ Create a new task that runs `executable` over a set of values (initially given by `initial_values_file`, then the output of a run is fed as input to the next one), iterating the process `total_iterations` times. If `slice_size` is a positive integer, then chop the input into chunks of -at most- the given size and compute them as separate independent jobs. Extra keyword arguments are saved and passed down to construct the `ValueFunctionIterationApplication`. """ assert slice_size >= 0, \ "Argument `slice_size` to ValueFunctionIteration.__init__" \ " must be a non-negative integer." # remember values for later use self.executable = executable self.initial_values = initial_values_file self.total_iterations = total_iterations - 1 self.slice_size = slice_size self.datadir = output_dir self.extra = extra_args # count initial values if os.path.exists(initial_values_file): f = open(initial_values_file, 'r') self.num_input_values = gc3libs.utils.count(f, lambda _: True) f.close() else: # XXX: there's no good default value! self.num_input_values = 0 # this little piece of black magic is to ensure intermediate # filenames appear numerically sorted in `ls -l` output self.values_filename_fmt = ('values.%%0%dd.txt' % (1 + int(math.log10(total_iterations)))) self.jobname = extra_args.get( 'jobname', gc3libs.utils.basename_sans(initial_values_file)) # create initial task and register it initial_task = ValueFunctionIterationPass(executable, initial_values_file, 0, total_iterations, slice_size, self.datadir, self.extra, parent=self.jobname) SequentialTaskCollection.__init__(self, self.jobname, [initial_task])
def __init__(self, jokes, **kwargs): self.jokes = jokes gc3libs.log.info("\t Calling MainSequentialFlow.__init({})".format(jokes)) self.initial_task = A(jokes) SequentialTaskCollection.__init__(self, [self.initial_task], **kwargs)
def __init__(self, joke, **kwargs): self.joke = joke gc3libs.log.info("\t\t\t\tCalling InnerSequentialFlow.__init__ for joke: {}".format(self.joke)) self.job_name = joke initial_task = B(joke) SequentialTaskCollection.__init__(self, [initial_task], **kwargs)
def __init__(self, name, ABC_UML_IMAGE, output, g3cfile, dimensions, fortran_pes, inputfilelist_abc, **extra_args): self.inputfilelist_abc = inputfilelist_abc self.output_folder = output self.extra_args = extra_args self.name = name # first step --> __init__ source compiling first_task = Gfit3C_ABC_uml_Application(abc_uml_image_file=ABC_UML_IMAGE, output_folder=output, g3c_input_file=g3cfile, dimension_file=dimensions, surface_file=fortran_pes, **extra_args) SequentialTaskCollection.__init__(self, name, [first_task])
def __init__(self, param_value, input_file_name, output_folder, iteration, **extra_args): gc3libs.log.info("\t\t\t\tCalling InnerSequentialIterationB.__init__ for param [%d] and file [%s]" % (param_value, input_file_name)) self.param_value = param_value self.input_file = input_file_name self.output_folder = output_folder self.jobname = "Gdemo_InnerSequenceB_"+str(self.param_value) initial_task = XtandemApplicationB(param_value, input_file_name, output_folder, iteration) SequentialTaskCollection.__init__(self, [initial_task], **extra_args)
def __init__(self, param_value, inputfile_folder, output_folder, **extra_args): self.param_value = param_value self.inputfile_folder = inputfile_folder self.output_folder = output_folder self.jobname = "Gdemo_MainSequence_"+str(self.param_value) gc3libs.log.info("\t Calling MainSequentialIteration.__init(%d,%s,%s)" % (param_value,inputfile_folder,output_folder)) self.initial_task = MainParallelIteration(param_value,inputfile_folder,output_folder) SequentialTaskCollection.__init__(self, [self.initial_task], **extra_args)
def __init__(self, executable, initial_values_file, total_iterations, slice_size=0, output_dir=TMPDIR, **extra_args): """ Create a new task that runs `executable` over a set of values (initially given by `initial_values_file`, then the output of a run is fed as input to the next one), iterating the process `total_iterations` times. If `slice_size` is a positive integer, then chop the input into chunks of -at most- the given size and compute them as separate independent jobs. Extra keyword arguments are saved and passed down to construct the `ValueFunctionIterationApplication`. """ assert slice_size >= 0, \ "Argument `slice_size` to ValueFunctionIteration.__init__" \ " must be a non-negative integer." # remember values for later use self.executable = executable self.initial_values = initial_values_file self.total_iterations = total_iterations - 1 self.slice_size = slice_size self.datadir = output_dir self.extra = extra_args # count initial values if os.path.exists(initial_values_file): f = open(initial_values_file, 'r') self.num_input_values = gc3libs.utils.count(f, lambda _: True) f.close() else: # XXX: there's no good default value! self.num_input_values = 0 # this little piece of black magic is to ensure intermediate # filenames appear numerically sorted in `ls -l` output self.values_filename_fmt = ('values.%%0%dd.txt' % (1 + int(math.log10(total_iterations)))) self.jobname = extra_args.get('jobname', gc3libs.utils.basename_sans(initial_values_file)) # create initial task and register it initial_task = ValueFunctionIterationPass(executable, initial_values_file, 0, total_iterations, slice_size, self.datadir, self.extra, parent=self.jobname) SequentialTaskCollection.__init__(self, self.jobname, [initial_task])
def __init__(self, param_value, input_file_name, output_folder, iteration, **extra_args): gc3libs.log.info( "\t\t\t\tCalling InnerSequentialIterationB.__init__ for param [%d] and file [%s]" % (param_value, input_file_name)) self.param_value = param_value self.input_file = input_file_name self.output_folder = output_folder self.jobname = "Gdemo_InnerSequenceB_" + str(self.param_value) initial_task = XtandemApplicationB(param_value, input_file_name, output_folder, iteration) SequentialTaskCollection.__init__(self, [initial_task], **extra_args)
def __init__(self, **kwargs): config = kwargs["config"] self.kwargs = kwargs gc3libs.log.info("\t Calling MainSequentialFlow.__init({})".format( "<No parameters>")) self.initial_tasks = [] if config["file_preparation"]["activated"] == 'True': self.initial_tasks = [ FilePreparation(name="file_preparation", **kwargs) ] else: self.initial_tasks = [SequencewiseParallelFlow(**kwargs)] SequentialTaskCollection.__init__(self, self.initial_tasks, **kwargs)
def __init__(self, input_bam_folder, **extra_args): self.gatkseq = extra_args['gatk_sequence'] self.S0_output = extra_args['S0_output'] self.S1_output = extra_args['S1_output'] self.input_bam_folder = input_bam_folder self.name = os.path.basename(input_bam_folder) self.extra = extra_args self.output_dir = extra_args['output_dir'] self.jobname = "GATK-" + "_".join(self.gatkseq) # Get the first stage task in gatk_sequence initial_task_fn = getattr(self, "stage%d" % int(self.gatkseq.pop(0))) initial_task = initial_task_fn() SequentialTaskCollection.__init__(self, [initial_task])
def __init__(self, input_image, copies, ncolors, size=None, **extra_args): """XXX do we need input_image and output_image? I guess so?""" self.input_image = input_image self.output_image = "warhol_%s" % os.path.basename(input_image) self.resize = False gc3libs.log.info( "Producing a warholized version of input file %s " "and store it in %s" % (input_image, self.output_image) ) if size: x, y = size.split("x", 2) rows = math.sqrt(copies) self.resize = "%dx%d" % (int(x) / rows, int(y) / rows) self.output_dir = os.path.relpath(extra_args.get("output_dir")) self.ncolors = ncolors self.copies = copies # Check that copies is a perfect square if math.sqrt(self.copies) != int(math.sqrt(self.copies)): raise gc3libs.exceptions.InvalidArgument("`copies` argument must be a perfect square.") self.jobname = extra_args.get("jobname", "WarholizedWorkflow") self.grayscaled_image = "grayscaled_%s" % os.path.basename(self.input_image) # This is just parsing of the arguments. The last lines, instead, # create the initial tasks that will be submitted. By now, we can submit # only the first one, `GrayScaleConvertApplication`, which will produce # a grayscale image from the input file:: self.extra_args = extra_args.copy() self.extra_args.pop("output_dir") self.tasks = [ GrayScaleConvertApplication( self.input_image, self.grayscaled_image, self.output_dir, self.output_dir, resize=self.resize, **self.extra_args ) ] SequentialTaskCollection.__init__(self, self.tasks)
def __init__(self, input_image, copies, ncolors, size=None, **extra_args): """XXX do we need input_image and output_image? I guess so?""" self.input_image = input_image self.output_image = "warhol_%s" % os.path.basename(input_image) self.resize = False gc3libs.log.info("Producing a warholized version of input file %s " "and store it in %s" % (input_image, self.output_image)) if size: x, y = size.split('x', 2) rows = math.sqrt(copies) self.resize = "%dx%d" % (int(x) / rows, int(y) / rows) self.output_dir = os.path.relpath(extra_args.get('output_dir')) self.ncolors = ncolors self.copies = copies # Check that copies is a perfect square if math.sqrt(self.copies) != int(math.sqrt(self.copies)): raise gc3libs.exceptions.InvalidArgument( "`copies` argument must be a perfect square.") self.jobname = extra_args.get('jobname', 'WarholizedWorkflow') self.grayscaled_image = "grayscaled_%s" % os.path.basename( self.input_image) # This is just parsing of the arguments. The last lines, instead, # create the initial tasks that will be submitted. By now, we can submit # only the first one, `GrayScaleConvertApplication`, which will produce # a grayscale image from the input file:: self.extra_args = extra_args.copy() self.extra_args.pop('output_dir') self.tasks = [ GrayScaleConvertApplication(self.input_image, self.grayscaled_image, self.output_dir, self.output_dir, resize=self.resize, **self.extra_args), ] SequentialTaskCollection.__init__(self, self.tasks)
def __init__(self, name, ABC_UML_IMAGE, output, g3cfile, dimensions, fortran_pes, inputfilelist_abc, **extra_args): self.inputfilelist_abc = inputfilelist_abc self.output_folder = output self.extra_args = extra_args self.name = name # first step --> __init__ source compiling first_task = Gfit3C_ABC_uml_Application( abc_uml_image_file=ABC_UML_IMAGE, output_folder=output, g3c_input_file=g3cfile, dimension_file=dimensions, surface_file=fortran_pes, **extra_args) SequentialTaskCollection.__init__(self, name, [first_task])
def __init__(self, **kwargs): config = kwargs["config"] self.kwargs = kwargs gc3libs.log.info( "\t Calling MainSequentialFlow.__init({})".format("<No parameters>")) self.initial_tasks = [] if config["file_preparation"]["activated"] == 'True': self.initial_tasks = [ FilePreparation( name="file_preparation", **kwargs)] else: self.initial_tasks = [SequencewiseParallelFlow(**kwargs)] SequentialTaskCollection.__init__(self, self.initial_tasks, **kwargs)
def __init__(self, param_value, inputfile_folder, output_folder, **extra_args): self.param_value = param_value self.inputfile_folder = inputfile_folder self.output_folder = output_folder self.jobname = "Gdemo_MainSequence_" + str(self.param_value) gc3libs.log.info( "\t Calling MainSequentialIteration.__init(%d,%s,%s)" % (param_value, inputfile_folder, output_folder)) self.initial_task = MainParallelIteration(param_value, inputfile_folder, output_folder) SequentialTaskCollection.__init__(self, [self.initial_task], **extra_args)
def __init__(self, repeats, cmdline, **extra_args): """ Initialize the repeated runs sequence. :param int repeats: Number of repetitions; if 0, repeat indefinitely. :param List[str] cmdline: Command-line to run. Can be a list of (string) arguments, or a single string to be passed to the shell to be interpreted as a command. """ self.repeats = repeats self.cmdline = cmdline # create initial task and register it initial_task = _CommandLineApp(0, self.cmdline) SequentialTaskCollection.__init__(self, [initial_task], **extra_args)
def __init__(self, substs, solverParas, **sessionParas): logger.debug('entering solveParaCombination.__init__ for job %s' % sessionParas['jobname']) self.iter = 0 self.jobname = 'solverParacombination' + sessionParas['jobname'] self.substs = substs self.sessionParas = sessionParas self.pathToExecutable = sessionParas['pathToExecutable'] self.architecture = sessionParas['architecture'] self.localBaseDir = sessionParas['localBaseDir'] self.paraFolder = os.path.join(os.getcwd(), sessionParas['jobname']) self.wBarLower_task = idRiskParaSearchDriver(self.paraFolder, self.substs, solverParas, **sessionParas) SequentialTaskCollection.__init__(self, self.jobname, [self.wBarLower_task]) logger.debug('done gParaSearchDriver.__init__ for job %s' % sessionParas['jobname'])
def __init__(self, grayscaled_image, output_dir, output_file, colors, warhol_dir, **extra_args): self.grayscaled_image = grayscaled_image self.output_dir = output_dir self.warhol_dir = warhol_dir self.jobname = 'TricolorizeImage' self.output_file = output_file if not os.path.isdir(output_dir): os.mkdir(output_dir) self.extra_args = extra_args gc3libs.log.info("Tricolorize image %s to %s" % (self.grayscaled_image, self.output_file)) self.tasks = [ CreateLutApplication(self.grayscaled_image, "%s.miff" % self.grayscaled_image, self.output_dir, colors, self.warhol_dir, **extra_args), ] SequentialTaskCollection.__init__(self, self.tasks)
def __init__(self, jobname='', path_to_stage_dir='', opt_algorithm=None, task_constructor=None, extract_value_fn=(lambda app: app.value), cur_pop_file = '', **extra_args): gc3libs.log.debug('entering ParallelDriver.__init__') # Set up initial variables and set the correct methods. self.jobname = jobname self.path_to_stage_dir = path_to_stage_dir self.opt_algorithm = opt_algorithm self.extract_value_fn = extract_value_fn self.task_constructor = task_constructor self.cur_pop_file = cur_pop_file self.extra_args = extra_args self.new_pop = self.opt_algorithm.pop initial_task = ComputeTargetVals( self.opt_algorithm.pop, self.jobname, self.opt_algorithm.cur_iter, path_to_stage_dir, self.cur_pop_file, task_constructor) SequentialTaskCollection.__init__(self, [initial_task], **extra_args)
def __init__(self, single_run_duration, generations_to_do, N, p_mut_coeff, choose_or_rand, sick_or_not, off_v_last, output_dir, executable=None, **extra_args): """Create a new task running an ``MHC_coev`` binary. Each binary is expected to run for `single_run_duration` minutes and dump its state in file ``latest_work.mat`` if it's not finished. This task will continue re-submitting the same executable together with the saved workspace until `generations_to_do` generations have been computed. :param single_run_duration: Duration of a single step (as a `gc3libs.quantity.Duration`:class: value) :param int generations_to_do: Count of generations that ``MHC_coev`` should simulate. :param N: Passed unchanged to the MHC_coev program. :param p_mut_coeff: Passed unchanged to the MHC_coev program. :param choose_or_rand: Passed unchanged to the MHC_coev program. :param sick_or_not: Passed unchanged to the MHC_coev program. :param off_v_last: Passed unchanged to the MHC_coev program. :param str output_dir: Path to a directory where output files from all runs should be collected. :param str executable: Path to the ``MHC_coev`` executable binary, or `None` (default) to specify that the default version available on the execution site should be used. """ # remember values for later use self.executable = executable self.output_dir = output_dir # allow 5 extra minutes for final saving the MatLab workspace self.single_run_duration = single_run_duration + 5 * minutes self.generations_to_do = generations_to_do self.p_mut_coeff = p_mut_coeff self.N = N self.choose_or_rand = choose_or_rand self.sick_or_not = sick_or_not self.off_v_last = off_v_last self.extra = extra_args self.generations_done = 0 # make up a sensibel job name if there isn't one if 'jobname' in extra_args: self.jobname = extra_args['jobname'] else: if self.executable is None: self.jobname = ( 'MHC_coev.%s.%s.%s.%s.%s' % (N, p_mut_coeff, choose_or_rand, sick_or_not, off_v_last)) else: os.path.basename(self.executable) # remove potentially conflicting kyword arguments extra_args.pop('output_dir', None) extra_args.pop('requested_walltime', None) # create initial task and register it initial_task = GMhcCoevApplication( N, p_mut_coeff, choose_or_rand, sick_or_not, off_v_last, output_dir=os.path.join(output_dir, 'tmp'), executable=self.executable, requested_walltime=self.single_run_duration, **extra_args) SequentialTaskCollection.__init__(self, [initial_task])
def __init__(self, paraFolder, substs, solverParas, **sessionParas): logger.debug('entering solveParaCombination.__init__ for job %s' % sessionParas['jobname']) self.iter = 0 self.jobname = 'idRiskParaSearchDriver' + sessionParas['jobname'] self.substs = substs self.sessionParas = sessionParas self.pathToExecutable = sessionParas['pathToExecutable'] self.architecture = sessionParas['architecture'] self.localBaseDir = sessionParas['localBaseDir'] self.paraFolder = os.path.join(os.getcwd(), sessionParas['jobname']) # setup AppPot parameters use_apppot = False apppot_img = None apppot_changes = None apppot_file = sessionParas['AppPotFile'] if apppot_file: use_apppot = True if apppot_file.endswith('.changes.tar.gz'): apppot_changes = apppot_file else: apppot_img = apppot_file pathToExecutable = sessionParas['pathToExecutable'] localBaseDir = sessionParas['localBaseDir'] architecture = sessionParas['architecture'] jobname = sessionParas['jobname'] executable = os.path.basename(self.pathToExecutable) inputs = { self.pathToExecutable:executable } # make a "stage" directory where input files are collected path_to_stage_dir = os.path.join(self.paraFolder, jobname) gc3libs.utils.mkdir(path_to_stage_dir) prefix_len = len(path_to_stage_dir) + 1 # 2. apply substitutions to parameter files for (path, changes) in substs.iteritems(): for (var, val, index, regex) in changes: support.update_parameter_in_file(os.path.join(localBaseDir, path), var, index, val, regex) support.fillInputDir(localBaseDir, path_to_stage_dir) # 3. build input file list for dirpath,dirnames,filenames in os.walk(path_to_stage_dir): for filename in filenames: # cut the leading part, which is == to path_to_stage_dir relpath = dirpath[prefix_len:] # ignore output directory contents in resubmission if relpath. startswith('output'): continue remote_path = os.path.join(relpath, filename) inputs[os.path.join(dirpath, filename)] = remote_path # all contents of the `output` directory are to be fetched outputs = { 'output/':'' } kwargs = {} kwargs['stdout'] = 'idRisk.log' kwargs['join'] = True kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output') kwargs['requested_architecture'] = self.architecture # adaptions for uml if use_apppot: if apppot_img is not None: kwargs['apppot_img'] = apppot_img if apppot_changes is not None: kwargs['apppot_changes'] = apppot_changes cls = idRiskApppotApplication else: cls = idRiskApplication kwargs.setdefault('tags', [ ]) # hand over job to create # self.curApplication = cls('/home/user/job/' + executable, [], inputs, outputs, **kwargs) self.curApplication = cls('./' + executable, [], inputs, outputs, **kwargs) SequentialTaskCollection.__init__(self, self.jobname, [ self.curApplication ]) logger.debug('done gParaSearchDriver.__init__ for job %s' % sessionParas['jobname'])
def __init__(self, *args, **kwargs): SequentialTaskCollection.__init__(self, *args, **kwargs) self.next_called_n_times = 0
def __init__(self, num_tasks, **extra_args): tasks = [ SuccessfulApp('stage{n}'.format(n=n)) for n in range(num_tasks) ] SequentialTaskCollection.__init__(self, tasks, **extra_args)
def __init__(self, pathToExecutable, pathToStageDir, architecture, baseDir, xVars, initialPop, nPopulation, domain, solverVerb, problemType, pathEmpirical, itermax, xConvCrit, yConvCrit, makePlots, optStrategy, fWeight, fCritical, ctryList, analyzeResults, nlc, plot3dTable, combOverviews, output_dir = '/tmp', **extra_args): ''' pathToExecutable: Path to main executable. pathToStageDir : Path to directory in which the action takes place. Usually set to os.getcwd(). architecture : Set the architecture used. baseDir : Directory in which the input files are assembled. This directory is sent as input to the cluseter. xVars : Names of the x variables. initialPop : The initial population if recovering from earlier failure. nPopulation : Population size. domain : The domain of the x variables. List of (lowerbound, upperbound) tuples. solverVerb : Verbosity of the solver. problemType : The problem type of the forward premium optimization. One4all, one4each etc. pathEmpirical : Path to the empirical results of the forward premium project. itermax : Maximum # of iterations of the solver. xConvCrit : Convergence criteria for x variables. yConvCrit : Convergence criteria for the y variables. makePlots : Make plots? optStrategy : The kind of differential evolution strategy to use. fWeight : fCritical : ctryList : The list of ctrys analyzed. analyzeResults : Function to analyze the output retrieved from the servers. nlc : Constraint function. plot3dTable : Function to generate 3d plots. combOverviews : Function to combine overviews. ''' logger.debug('entering gParaSearchDriver.__init__') # Set up initial variables and set the correct methods. self.pathToStageDir = pathToStageDir self.problemType = problemType self.pathToExecutable = pathToExecutable self.architecture = architecture self.baseDir = baseDir self.verbosity = solverVerb.upper() self.jobname = extra_args['jobname'] self.ctryList = ctryList.split() self.xVars = xVars self.domain = domain self.n = len(self.xVars.split()) self.extra_args = extra_args self.analyzeResults = analyzeResults self.plot3dTable = plot3dTable self.combOverviews = combOverviews # Set solver options S_struct = {} S_struct['I_NP'] = int(nPopulation) S_struct['F_weight'] = float(fWeight) S_struct['F_CR'] = float(fCritical) S_struct['I_D'] = self.n S_struct['lowerBds'] = np.array([ element[0] for element in domain ], dtype = 'float64') S_struct['upperBds'] = np.array([ element[1] for element in domain ], dtype = 'float64') S_struct['I_itermax'] = int(itermax) S_struct['F_VTR'] = float(yConvCrit) S_struct['I_strategy'] = int(optStrategy) S_struct['I_plotting'] = int(makePlots) S_struct['xConvCrit'] = float(xConvCrit) S_struct['workingDir'] = pathToStageDir S_struct['verbosity'] = self.verbosity # Initialize solver self.deSolver = deKenPrice(S_struct) # Set constraint function in solver. self.deSolver.nlc = nlc # create initial task and register it if initialPop: self.deSolver.newPop = np.loadtxt(initialPop, delimiter = ' ') else: self.deSolver.newPop = self.deSolver.drawInitialSample() #self.deSolver.plotPopulation(self.deSolver.newPop) self.deSolver.I_iter += 1 self.evaluator = gParaSearchParallel(self.deSolver.newPop, self.deSolver.I_iter, self.pathToExecutable, self.pathToStageDir, self.architecture, self.baseDir, self.xVars, self.verbosity, self.problemType, self.analyzeResults, self.ctryList, **self.extra_args) initial_task = self.evaluator SequentialTaskCollection.__init__(self, self.jobname, [initial_task])
def __init__(self, single_run_duration, generations_to_do, N, p_mut_coeff, choose_or_rand, sick_or_not, off_v_last, output_dir, executable=None, **extra_args): """Create a new task running an ``MHC_coev`` binary. Each binary is expected to run for `single_run_duration` minutes and dump its state in file ``latest_work.mat`` if it's not finished. This task will continue re-submitting the same executable together with the saved workspace until `generations_to_do` generations have been computed. :param single_run_duration: Duration of a single step (as a `gc3libs.quantity.Duration`:class: value) :param int generations_to_do: Count of generations that ``MHC_coev`` should simulate. :param N: Passed unchanged to the MHC_coev program. :param p_mut_coeff: Passed unchanged to the MHC_coev program. :param choose_or_rand: Passed unchanged to the MHC_coev program. :param sick_or_not: Passed unchanged to the MHC_coev program. :param off_v_last: Passed unchanged to the MHC_coev program. :param str output_dir: Path to a directory where output files from all runs should be collected. :param str executable: Path to the ``MHC_coev`` executable binary, or `None` (default) to specify that the default version available on the execution site should be used. """ # remember values for later use self.executable = executable self.output_dir = output_dir # allow 5 extra minutes for final saving the MatLab workspace self.single_run_duration = single_run_duration + 5*minutes self.generations_to_do = generations_to_do self.p_mut_coeff = p_mut_coeff self.N = N self.choose_or_rand = choose_or_rand self.sick_or_not = sick_or_not self.off_v_last = off_v_last self.extra = extra_args self.generations_done = 0 # make up a sensibel job name if there isn't one if 'jobname' in extra_args: self.jobname = extra_args['jobname'] else: if self.executable is None: self.jobname = ('MHC_coev.%s.%s.%s.%s.%s' % (N, p_mut_coeff, choose_or_rand, sick_or_not, off_v_last)) else: os.path.basename(self.executable) # remove potentially conflicting kyword arguments extra_args.pop('output_dir', None) extra_args.pop('requested_walltime', None) # create initial task and register it initial_task = GMhcCoevApplication( N, p_mut_coeff, choose_or_rand, sick_or_not, off_v_last, output_dir = os.path.join(output_dir, 'tmp'), executable = self.executable, requested_walltime = self.single_run_duration, **extra_args) SequentialTaskCollection.__init__(self, [initial_task])
def __init__(self, num_tasks, **extra_args): tasks = [SuccessfulApp('stage{n}'.format(n=n)) for n in range(num_tasks)] SequentialTaskCollection.__init__(self, tasks, **extra_args)