def __init__(self, videofile, email_from, smtp_server, **extra):
self.videofile = videofile
videofilename = os.path.basename(videofile)
inboxdir = os.path.dirname(videofile)
self.vdescrfile = os.path.join(inboxdir, 'video.description.txt')
self.inboxdir = inboxdir
self.email_to = ''
self.email_from = email_from
self.smtp_server = smtp_server
self.video_is_needed = True
# Look for a 'configuration' file in the inboxdir to override
# bemovi parameters
videoname = videofilename.rsplit('.', 1)[0]
self.extra = extra
extra['jobname'] = 'BemoviWorkflow_%s' % videoname
extra['videoname'] = videoname
extra['base_output_dir'] = os.path.join(
inboxdir + '.out',
extra['jobname'])
self.update_configuration()
plocator = ParticleLocator(videofile, **extra)
plinker = ParticleLinker(
os.path.join(extra['base_output_dir'],
ParticleLocator.application,
extra['videoname'] + '.ijout.txt',),
**extra)
SequentialTaskCollection.__init__(self, [plocator, plinker], **extra)
def __init__(self, grayscaled_image, output_dir, output_file,
colors, warhol_dir, **extra_args):
self.grayscaled_image = grayscaled_image
self.output_dir = output_dir
self.warhol_dir = warhol_dir
self.jobname = 'TricolorizeImage'
self.output_file = output_file
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
self.extra_args = extra_args
gc3libs.log.info(
"Tricolorize image %s to %s" % (
self.grayscaled_image, self.output_file))
self.tasks = [
CreateLutApplication(
self.grayscaled_image,
"%s.miff" % self.grayscaled_image,
self.output_dir,
colors, self.warhol_dir, **extra_args),
]
SequentialTaskCollection.__init__(self, self.tasks)
def __init__(self,
jobname='',
path_to_stage_dir='',
opt_algorithm=None,
task_constructor=None,
extract_value_fn=(lambda app: app.value),
cur_pop_file='',
**extra_args):
gc3libs.log.debug('entering ParallelDriver.__init__')
# Set up initial variables and set the correct methods.
self.jobname = jobname
self.path_to_stage_dir = path_to_stage_dir
self.opt_algorithm = opt_algorithm
self.extract_value_fn = extract_value_fn
self.task_constructor = task_constructor
self.cur_pop_file = cur_pop_file
self.extra_args = extra_args
self.new_pop = self.opt_algorithm.pop
initial_task = ComputeTargetVals(self.opt_algorithm.pop, self.jobname,
self.opt_algorithm.cur_iter,
path_to_stage_dir, self.cur_pop_file,
task_constructor)
SequentialTaskCollection.__init__(self, [initial_task], **extra_args)
def __init__(self, substs, solverParas, **sessionParas):
logger.debug('entering solveParaCombination.__init__ for job %s' %
sessionParas['jobname'])
self.iter = 0
self.jobname = sessionParas['jobname']
self.substs = substs
self.sessionParas = sessionParas
self.pathToExecutable = sessionParas['pathToExecutable']
self.architecture = sessionParas['architecture']
self.localBaseDir = sessionParas['localBaseDir']
self.paraFolder = os.path.join(os.getcwd(), sessionParas['jobname'])
self.wBarLower_task = idRiskParaSearchDriver(self.paraFolder,
self.substs, solverParas,
**sessionParas)
SequentialTaskCollection.__init__(self, self.jobname,
[self.wBarLower_task])
logger.debug('done gParaSearchDriver.__init__ for job %s' %
sessionParas['jobname'])
def __init__(self, init_value, add_value, iterations, **extra_args):
"""
Create a new tasks that runs `executable` over a set of values
(initially given by `initial_values_file`, then the output of
a run is fed as input to the next one), riterating the process
`total_iterations` times.
If `slice_size` is a positive integer, then chop the input into
chunks of -at most- the given size and compute them as separate
independent jobs.
Extra keyword arguments are saved and passed down to construct
the `ValueFunctionIterationApplication`.
"""
self.init = init_value
self.increment = add_value
self.limit = iterations
self.jobname = "Gdemo_Iteration"
gc3libs.log.info("Calling DemoIteration.__init__() ... ")
# create initial task and register it
initial_task = GdemoApplication(self.init, self.increment, 0)
SequentialTaskCollection.__init__(self, [initial_task], **extra_args)
def __init__(self, paraFolder, substs, solverParas, **sessionParas):
logger.debug('entering gParaSearchDriver.__init__')
self.jobname = sessionParas['jobname'] + 'driver'
solverParas['jobname'] = self.jobname
self.sessionParas = sessionParas
self.solverParas = solverParas
self.substs = substs
self.paraFolder = paraFolder
self.iter = 0
# create a subfolder for optim over xVar
self.optimFolder = os.path.join(paraFolder, 'optim' + str(solverParas['xVars'][0]))
self.solverParas['optimFolder'] = self.optimFolder
gc3libs.utils.mkdir(self.optimFolder)
self.costlyOptimizer = costlyOptimization.costlyOptimization(solverParas)
self.xVars = solverParas['xVars']
self.xParaCombos = solverParas['xInitialParaCombo']
self.targetVars = solverParas['targetVars']
self.target_fx = solverParas['target_fx']
self.evaluator = idRiskParaSearchParallel(self.xVars, self.xParaCombos, substs, self.optimFolder, solverParas , **sessionParas)
initial_task = self.evaluator
SequentialTaskCollection.__init__(self, self.jobname, [initial_task])
logger.debug('done solveParaCombination.__init__')
def __init__(self, paraFolder, substs, solverParas, **sessionParas):
logger.debug('entering gParaSearchDriver.__init__')
self.jobname = sessionParas['jobname'] + 'driver'
solverParas['jobname'] = self.jobname
self.sessionParas = sessionParas
self.solverParas = solverParas
self.substs = substs
self.paraFolder = paraFolder
self.iter = 0
# create a subfolder for optim over xVar
self.optimFolder = os.path.join(paraFolder, 'optim' + str(solverParas['xVars'][0]))
gc3libs.utils.mkdir(self.optimFolder)
self.costlyOptimizer = costlyOptimization.costlyOptimization(solverParas)
self.xVars = solverParas['xVars']
self.xParaCombos = solverParas['xInitialParaCombo']
self.targetVars = solverParas['targetVars']
self.target_fx = solverParas['target_fx']
self.evaluator = idRiskParaSearchParallel(self.xVars, self.xParaCombos, substs, self.optimFolder, solverParas , **sessionParas)
initial_task = self.evaluator
SequentialTaskCollection.__init__(self, self.jobname, [initial_task])
logger.debug('done solveParaCombination.__init__')
def __init__(self, init_value, add_value, iterations, **extra_args):
"""
Create a new tasks that runs `executable` over a set of values
(initially given by `initial_values_file`, then the output of
a run is fed as input to the next one), riterating the process
`total_iterations` times.
If `slice_size` is a positive integer, then chop the input into
chunks of -at most- the given size and compute them as separate
independent jobs.
Extra keyword arguments are saved and passed down to construct
the `ValueFunctionIterationApplication`.
"""
self.init = init_value
self.increment = add_value
self.limit = iterations
self.jobname = "Gdemo_Iteration"
gc3libs.log.info("Calling DemoIteration.__init__() ... ")
# create initial task and register it
initial_task = GdemoApplication(self.init, self.increment, 0)
SequentialTaskCollection.__init__(self, [initial_task], **extra_args)
def __init__(self, name, experiment_id, verbosity,
submission_id, user_name, parent_id, description=None,
waiting_time=0):
'''
Parameters
----------
name: str
name of the stage
experiment_id: int
ID of the processed experiment
verbosity: int
logging verbosity index
submission_id: int
ID of the corresponding submission
user_name: str
name of the submitting user
parent_id: int
ID of the parent
:class:`Workflow <tmlib.workflow.workflow.Workflow>`
description: tmlib.tmaps.description.WorkflowStageDescription, optional
description of the stage (default: ``None``)
waiting_time: int, optional
time in seconds that should be waited upon transition from one
stage to the other to avoid issues related to network file systems
(default: ``0``)
'''
SequentialTaskCollection.__init__(
self, tasks=None, jobname='%s' % name
)
WorkflowStage.__init__(
self, name=name, experiment_id=experiment_id, verbosity=verbosity,
submission_id=submission_id, description=description,
user_name=user_name, parent_id=parent_id
)
self.waiting_time = waiting_time
def __init__(self, name, experiment_id, verbosity,
submission_id, user_name, parent_id, description=None,
waiting_time=0):
'''
Parameters
----------
name: str
name of the stage
experiment_id: int
ID of the processed experiment
verbosity: int
logging verbosity index
submission_id: int
ID of the corresponding submission
user_name: str
name of the submitting user
parent_id: int
ID of the parent
:class:`Workflow <tmlib.workflow.workflow.Workflow>`
description: tmlib.tmaps.description.WorkflowStageDescription, optional
description of the stage (default: ``None``)
waiting_time: int, optional
time in seconds that should be waited upon transition from one
stage to the other to avoid issues related to network file systems
(default: ``0``)
'''
SequentialTaskCollection.__init__(
self, tasks=None, jobname='%s' % name
)
WorkflowStage.__init__(
self, name=name, experiment_id=experiment_id, verbosity=verbosity,
submission_id=submission_id, description=description,
user_name=user_name, parent_id=parent_id
)
self.waiting_time = waiting_time
def __init__(self, paraFolder, substs, solverParas, **sessionParas):
logger.debug("entering gParaSearchDriver.__init__")
self.jobname = sessionParas["jobname"] + "driver"
solverParas["jobname"] = self.jobname
self.sessionParas = sessionParas
self.solverParas = solverParas
self.substs = substs
self.paraFolder = paraFolder
self.iter = 0
# create a subfolder for optim over xVar
self.optimFolder = os.path.join(paraFolder, "optim" + str(solverParas["xVars"][0]))
self.solverParas["optimFolder"] = self.optimFolder
gc3libs.utils.mkdir(self.optimFolder)
self.costlyOptimizer = costlyOptimization.costlyOptimization(solverParas)
self.xVars = solverParas["xVars"]
self.xParaCombos = solverParas["xInitialParaCombo"]
self.targetVars = solverParas["targetVars"]
self.target_fx = solverParas["target_fx"]
self.evaluator = idRiskParaSearchParallel(
self.xVars, self.xParaCombos, substs, self.optimFolder, solverParas, **sessionParas
)
initial_task = self.evaluator
SequentialTaskCollection.__init__(self, self.jobname, [initial_task])
logger.debug("done solveParaCombination.__init__")
def __init__(self, videofile, email_from, smtp_server, **extra):
self.videofile = videofile
videofilename = os.path.basename(videofile)
inboxdir = os.path.dirname(videofile)
self.vdescrfile = os.path.join(inboxdir, 'video.description.txt')
self.inboxdir = inboxdir
self.email_to = ''
self.email_from = email_from
self.smtp_server = smtp_server
self.video_is_needed = True
# Look for a 'configuration' file in the inboxdir to override
# bemovi parameters
videoname = videofilename.rsplit('.', 1)[0]
self.extra = extra
extra['jobname'] = 'BemoviWorkflow_%s' % videoname
extra['videoname'] = videoname
extra['base_output_dir'] = os.path.join(inboxdir + '.out',
extra['jobname'])
self.update_configuration()
plocator = ParticleLocator(videofile, **extra)
plinker = ParticleLinker(
os.path.join(
extra['base_output_dir'],
ParticleLocator.application,
extra['videoname'] + '.ijout.txt',
), **extra)
SequentialTaskCollection.__init__(self, [plocator, plinker], **extra)
def __init__(self,
executable,
initial_values_file,
total_iterations,
slice_size=0,
output_dir=TMPDIR,
**extra_args):
"""
Create a new task that runs `executable` over a set of values
(initially given by `initial_values_file`, then the output of
a run is fed as input to the next one), iterating the process
`total_iterations` times.
If `slice_size` is a positive integer, then chop the input into
chunks of -at most- the given size and compute them as separate
independent jobs.
Extra keyword arguments are saved and passed down to construct
the `ValueFunctionIterationApplication`.
"""
assert slice_size >= 0, \
"Argument `slice_size` to ValueFunctionIteration.__init__" \
" must be a non-negative integer."
# remember values for later use
self.executable = executable
self.initial_values = initial_values_file
self.total_iterations = total_iterations - 1
self.slice_size = slice_size
self.datadir = output_dir
self.extra = extra_args
# count initial values
if os.path.exists(initial_values_file):
f = open(initial_values_file, 'r')
self.num_input_values = gc3libs.utils.count(f, lambda _: True)
f.close()
else:
# XXX: there's no good default value!
self.num_input_values = 0
# this little piece of black magic is to ensure intermediate
# filenames appear numerically sorted in `ls -l` output
self.values_filename_fmt = ('values.%%0%dd.txt' %
(1 + int(math.log10(total_iterations))))
self.jobname = extra_args.get(
'jobname', gc3libs.utils.basename_sans(initial_values_file))
# create initial task and register it
initial_task = ValueFunctionIterationPass(executable,
initial_values_file,
0,
total_iterations,
slice_size,
self.datadir,
self.extra,
parent=self.jobname)
SequentialTaskCollection.__init__(self, self.jobname, [initial_task])
def __init__(self, jokes, **kwargs):
self.jokes = jokes
gc3libs.log.info("\t Calling MainSequentialFlow.__init({})".format(jokes))
self.initial_task = A(jokes)
SequentialTaskCollection.__init__(self, [self.initial_task], **kwargs)
def __init__(self, joke, **kwargs):
self.joke = joke
gc3libs.log.info("\t\t\t\tCalling InnerSequentialFlow.__init__ for joke: {}".format(self.joke))
self.job_name = joke
initial_task = B(joke)
SequentialTaskCollection.__init__(self, [initial_task], **kwargs)
def __init__(self, name, ABC_UML_IMAGE, output, g3cfile, dimensions, fortran_pes, inputfilelist_abc, **extra_args):
self.inputfilelist_abc = inputfilelist_abc
self.output_folder = output
self.extra_args = extra_args
self.name = name
# first step --> __init__ source compiling
first_task = Gfit3C_ABC_uml_Application(abc_uml_image_file=ABC_UML_IMAGE, output_folder=output, g3c_input_file=g3cfile, dimension_file=dimensions, surface_file=fortran_pes, **extra_args)
SequentialTaskCollection.__init__(self, name, [first_task])
def __init__(self, param_value, input_file_name, output_folder, iteration, **extra_args):
gc3libs.log.info("\t\t\t\tCalling InnerSequentialIterationB.__init__ for param [%d] and file [%s]" % (param_value, input_file_name))
self.param_value = param_value
self.input_file = input_file_name
self.output_folder = output_folder
self.jobname = "Gdemo_InnerSequenceB_"+str(self.param_value)
initial_task = XtandemApplicationB(param_value, input_file_name, output_folder, iteration)
SequentialTaskCollection.__init__(self, [initial_task], **extra_args)
def __init__(self, param_value, inputfile_folder, output_folder, **extra_args):
self.param_value = param_value
self.inputfile_folder = inputfile_folder
self.output_folder = output_folder
self.jobname = "Gdemo_MainSequence_"+str(self.param_value)
gc3libs.log.info("\t Calling MainSequentialIteration.__init(%d,%s,%s)" % (param_value,inputfile_folder,output_folder))
self.initial_task = MainParallelIteration(param_value,inputfile_folder,output_folder)
SequentialTaskCollection.__init__(self, [self.initial_task], **extra_args)
def __init__(self, executable, initial_values_file,
total_iterations, slice_size=0,
output_dir=TMPDIR, **extra_args):
"""
Create a new task that runs `executable` over a set of values
(initially given by `initial_values_file`, then the output of
a run is fed as input to the next one), iterating the process
`total_iterations` times.
If `slice_size` is a positive integer, then chop the input into
chunks of -at most- the given size and compute them as separate
independent jobs.
Extra keyword arguments are saved and passed down to construct
the `ValueFunctionIterationApplication`.
"""
assert slice_size >= 0, \
"Argument `slice_size` to ValueFunctionIteration.__init__" \
" must be a non-negative integer."
# remember values for later use
self.executable = executable
self.initial_values = initial_values_file
self.total_iterations = total_iterations - 1
self.slice_size = slice_size
self.datadir = output_dir
self.extra = extra_args
# count initial values
if os.path.exists(initial_values_file):
f = open(initial_values_file, 'r')
self.num_input_values = gc3libs.utils.count(f, lambda _: True)
f.close()
else:
# XXX: there's no good default value!
self.num_input_values = 0
# this little piece of black magic is to ensure intermediate
# filenames appear numerically sorted in `ls -l` output
self.values_filename_fmt = ('values.%%0%dd.txt'
% (1 + int(math.log10(total_iterations))))
self.jobname = extra_args.get('jobname',
gc3libs.utils.basename_sans(initial_values_file))
# create initial task and register it
initial_task = ValueFunctionIterationPass(executable, initial_values_file,
0, total_iterations, slice_size,
self.datadir, self.extra,
parent=self.jobname)
SequentialTaskCollection.__init__(self, self.jobname, [initial_task])
def __init__(self, param_value, input_file_name, output_folder, iteration,
**extra_args):
gc3libs.log.info(
"\t\t\t\tCalling InnerSequentialIterationB.__init__ for param [%d] and file [%s]"
% (param_value, input_file_name))
self.param_value = param_value
self.input_file = input_file_name
self.output_folder = output_folder
self.jobname = "Gdemo_InnerSequenceB_" + str(self.param_value)
initial_task = XtandemApplicationB(param_value, input_file_name,
output_folder, iteration)
SequentialTaskCollection.__init__(self, [initial_task], **extra_args)
def __init__(self, **kwargs):
config = kwargs["config"]
self.kwargs = kwargs
gc3libs.log.info("\t Calling MainSequentialFlow.__init({})".format(
"<No parameters>"))
self.initial_tasks = []
if config["file_preparation"]["activated"] == 'True':
self.initial_tasks = [
FilePreparation(name="file_preparation", **kwargs)
]
else:
self.initial_tasks = [SequencewiseParallelFlow(**kwargs)]
SequentialTaskCollection.__init__(self, self.initial_tasks, **kwargs)
def __init__(self, input_bam_folder, **extra_args):
self.gatkseq = extra_args['gatk_sequence']
self.S0_output = extra_args['S0_output']
self.S1_output = extra_args['S1_output']
self.input_bam_folder = input_bam_folder
self.name = os.path.basename(input_bam_folder)
self.extra = extra_args
self.output_dir = extra_args['output_dir']
self.jobname = "GATK-" + "_".join(self.gatkseq)
# Get the first stage task in gatk_sequence
initial_task_fn = getattr(self, "stage%d" % int(self.gatkseq.pop(0)))
initial_task = initial_task_fn()
SequentialTaskCollection.__init__(self, [initial_task])
def __init__(self, input_bam_folder, **extra_args):
self.gatkseq = extra_args['gatk_sequence']
self.S0_output = extra_args['S0_output']
self.S1_output = extra_args['S1_output']
self.input_bam_folder = input_bam_folder
self.name = os.path.basename(input_bam_folder)
self.extra = extra_args
self.output_dir = extra_args['output_dir']
self.jobname = "GATK-" + "_".join(self.gatkseq)
# Get the first stage task in gatk_sequence
initial_task_fn = getattr(self, "stage%d" % int(self.gatkseq.pop(0)))
initial_task = initial_task_fn()
SequentialTaskCollection.__init__(self, [initial_task])
def __init__(self, input_image, copies, ncolors, size=None, **extra_args):
"""XXX do we need input_image and output_image? I guess so?"""
self.input_image = input_image
self.output_image = "warhol_%s" % os.path.basename(input_image)
self.resize = False
gc3libs.log.info(
"Producing a warholized version of input file %s " "and store it in %s" % (input_image, self.output_image)
)
if size:
x, y = size.split("x", 2)
rows = math.sqrt(copies)
self.resize = "%dx%d" % (int(x) / rows, int(y) / rows)
self.output_dir = os.path.relpath(extra_args.get("output_dir"))
self.ncolors = ncolors
self.copies = copies
# Check that copies is a perfect square
if math.sqrt(self.copies) != int(math.sqrt(self.copies)):
raise gc3libs.exceptions.InvalidArgument("`copies` argument must be a perfect square.")
self.jobname = extra_args.get("jobname", "WarholizedWorkflow")
self.grayscaled_image = "grayscaled_%s" % os.path.basename(self.input_image)
# This is just parsing of the arguments. The last lines, instead,
# create the initial tasks that will be submitted. By now, we can submit
# only the first one, `GrayScaleConvertApplication`, which will produce
# a grayscale image from the input file::
self.extra_args = extra_args.copy()
self.extra_args.pop("output_dir")
self.tasks = [
GrayScaleConvertApplication(
self.input_image,
self.grayscaled_image,
self.output_dir,
self.output_dir,
resize=self.resize,
**self.extra_args
)
]
SequentialTaskCollection.__init__(self, self.tasks)
def __init__(self, input_image, copies, ncolors, size=None, **extra_args):
"""XXX do we need input_image and output_image? I guess so?"""
self.input_image = input_image
self.output_image = "warhol_%s" % os.path.basename(input_image)
self.resize = False
gc3libs.log.info("Producing a warholized version of input file %s "
"and store it in %s" %
(input_image, self.output_image))
if size:
x, y = size.split('x', 2)
rows = math.sqrt(copies)
self.resize = "%dx%d" % (int(x) / rows, int(y) / rows)
self.output_dir = os.path.relpath(extra_args.get('output_dir'))
self.ncolors = ncolors
self.copies = copies
# Check that copies is a perfect square
if math.sqrt(self.copies) != int(math.sqrt(self.copies)):
raise gc3libs.exceptions.InvalidArgument(
"`copies` argument must be a perfect square.")
self.jobname = extra_args.get('jobname', 'WarholizedWorkflow')
self.grayscaled_image = "grayscaled_%s" % os.path.basename(
self.input_image)
# This is just parsing of the arguments. The last lines, instead,
# create the initial tasks that will be submitted. By now, we can submit
# only the first one, `GrayScaleConvertApplication`, which will produce
# a grayscale image from the input file::
self.extra_args = extra_args.copy()
self.extra_args.pop('output_dir')
self.tasks = [
GrayScaleConvertApplication(self.input_image,
self.grayscaled_image,
self.output_dir,
self.output_dir,
resize=self.resize,
**self.extra_args),
]
SequentialTaskCollection.__init__(self, self.tasks)
def __init__(self, name, ABC_UML_IMAGE, output, g3cfile, dimensions,
fortran_pes, inputfilelist_abc, **extra_args):
self.inputfilelist_abc = inputfilelist_abc
self.output_folder = output
self.extra_args = extra_args
self.name = name
# first step --> __init__ source compiling
first_task = Gfit3C_ABC_uml_Application(
abc_uml_image_file=ABC_UML_IMAGE,
output_folder=output,
g3c_input_file=g3cfile,
dimension_file=dimensions,
surface_file=fortran_pes,
**extra_args)
SequentialTaskCollection.__init__(self, name, [first_task])
def __init__(self, **kwargs):
config = kwargs["config"]
self.kwargs = kwargs
gc3libs.log.info(
"\t Calling MainSequentialFlow.__init({})".format("<No parameters>"))
self.initial_tasks = []
if config["file_preparation"]["activated"] == 'True':
self.initial_tasks = [
FilePreparation(
name="file_preparation",
**kwargs)]
else:
self.initial_tasks = [SequencewiseParallelFlow(**kwargs)]
SequentialTaskCollection.__init__(self, self.initial_tasks, **kwargs)
def __init__(self, param_value, inputfile_folder, output_folder,
**extra_args):
self.param_value = param_value
self.inputfile_folder = inputfile_folder
self.output_folder = output_folder
self.jobname = "Gdemo_MainSequence_" + str(self.param_value)
gc3libs.log.info(
"\t Calling MainSequentialIteration.__init(%d,%s,%s)" %
(param_value, inputfile_folder, output_folder))
self.initial_task = MainParallelIteration(param_value,
inputfile_folder,
output_folder)
SequentialTaskCollection.__init__(self, [self.initial_task],
**extra_args)
def __init__(self, repeats, cmdline, **extra_args):
"""
Initialize the repeated runs sequence.
:param int repeats:
Number of repetitions; if 0, repeat indefinitely.
:param List[str] cmdline:
Command-line to run. Can be a list of (string) arguments, or
a single string to be passed to the shell to be interpreted
as a command.
"""
self.repeats = repeats
self.cmdline = cmdline
# create initial task and register it
initial_task = _CommandLineApp(0, self.cmdline)
SequentialTaskCollection.__init__(self, [initial_task], **extra_args)
def __init__(self, substs, solverParas, **sessionParas):
logger.debug('entering solveParaCombination.__init__ for job %s' % sessionParas['jobname'])
self.iter = 0
self.jobname = 'solverParacombination' + sessionParas['jobname']
self.substs = substs
self.sessionParas = sessionParas
self.pathToExecutable = sessionParas['pathToExecutable']
self.architecture = sessionParas['architecture']
self.localBaseDir = sessionParas['localBaseDir']
self.paraFolder = os.path.join(os.getcwd(), sessionParas['jobname'])
self.wBarLower_task = idRiskParaSearchDriver(self.paraFolder, self.substs, solverParas, **sessionParas)
SequentialTaskCollection.__init__(self, self.jobname, [self.wBarLower_task])
logger.debug('done gParaSearchDriver.__init__ for job %s' % sessionParas['jobname'])
def __init__(self, grayscaled_image, output_dir, output_file, colors,
warhol_dir, **extra_args):
self.grayscaled_image = grayscaled_image
self.output_dir = output_dir
self.warhol_dir = warhol_dir
self.jobname = 'TricolorizeImage'
self.output_file = output_file
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
self.extra_args = extra_args
gc3libs.log.info("Tricolorize image %s to %s" %
(self.grayscaled_image, self.output_file))
self.tasks = [
CreateLutApplication(self.grayscaled_image,
"%s.miff" % self.grayscaled_image,
self.output_dir, colors, self.warhol_dir,
**extra_args),
]
SequentialTaskCollection.__init__(self, self.tasks)
def __init__(self, jobname='', path_to_stage_dir='',
opt_algorithm=None, task_constructor=None,
extract_value_fn=(lambda app: app.value),
cur_pop_file = '',
**extra_args):
gc3libs.log.debug('entering ParallelDriver.__init__')
# Set up initial variables and set the correct methods.
self.jobname = jobname
self.path_to_stage_dir = path_to_stage_dir
self.opt_algorithm = opt_algorithm
self.extract_value_fn = extract_value_fn
self.task_constructor = task_constructor
self.cur_pop_file = cur_pop_file
self.extra_args = extra_args
self.new_pop = self.opt_algorithm.pop
initial_task = ComputeTargetVals(
self.opt_algorithm.pop, self.jobname, self.opt_algorithm.cur_iter,
path_to_stage_dir, self.cur_pop_file, task_constructor)
SequentialTaskCollection.__init__(self, [initial_task], **extra_args)
def __init__(self,
single_run_duration,
generations_to_do,
N,
p_mut_coeff,
choose_or_rand,
sick_or_not,
off_v_last,
output_dir,
executable=None,
**extra_args):
"""Create a new task running an ``MHC_coev`` binary.
Each binary is expected to run for `single_run_duration`
minutes and dump its state in file ``latest_work.mat`` if it's
not finished. This task will continue re-submitting the same
executable together with the saved workspace until
`generations_to_do` generations have been computed.
:param single_run_duration:
Duration of a single step (as a `gc3libs.quantity.Duration`:class: value)
:param int generations_to_do:
Count of generations that ``MHC_coev`` should simulate.
:param N: Passed unchanged to the MHC_coev program.
:param p_mut_coeff: Passed unchanged to the MHC_coev program.
:param choose_or_rand: Passed unchanged to the MHC_coev program.
:param sick_or_not: Passed unchanged to the MHC_coev program.
:param off_v_last: Passed unchanged to the MHC_coev program.
:param str output_dir:
Path to a directory where output files
from all runs should be collected.
:param str executable:
Path to the ``MHC_coev`` executable binary, or `None`
(default) to specify that the default version available on
the execution site should be used.
"""
# remember values for later use
self.executable = executable
self.output_dir = output_dir
# allow 5 extra minutes for final saving the MatLab workspace
self.single_run_duration = single_run_duration + 5 * minutes
self.generations_to_do = generations_to_do
self.p_mut_coeff = p_mut_coeff
self.N = N
self.choose_or_rand = choose_or_rand
self.sick_or_not = sick_or_not
self.off_v_last = off_v_last
self.extra = extra_args
self.generations_done = 0
# make up a sensibel job name if there isn't one
if 'jobname' in extra_args:
self.jobname = extra_args['jobname']
else:
if self.executable is None:
self.jobname = (
'MHC_coev.%s.%s.%s.%s.%s' %
(N, p_mut_coeff, choose_or_rand, sick_or_not, off_v_last))
else:
os.path.basename(self.executable)
# remove potentially conflicting kyword arguments
extra_args.pop('output_dir', None)
extra_args.pop('requested_walltime', None)
# create initial task and register it
initial_task = GMhcCoevApplication(
N,
p_mut_coeff,
choose_or_rand,
sick_or_not,
off_v_last,
output_dir=os.path.join(output_dir, 'tmp'),
executable=self.executable,
requested_walltime=self.single_run_duration,
**extra_args)
SequentialTaskCollection.__init__(self, [initial_task])
def __init__(self, paraFolder, substs, solverParas, **sessionParas):
logger.debug('entering solveParaCombination.__init__ for job %s' % sessionParas['jobname'])
self.iter = 0
self.jobname = 'idRiskParaSearchDriver' + sessionParas['jobname']
self.substs = substs
self.sessionParas = sessionParas
self.pathToExecutable = sessionParas['pathToExecutable']
self.architecture = sessionParas['architecture']
self.localBaseDir = sessionParas['localBaseDir']
self.paraFolder = os.path.join(os.getcwd(), sessionParas['jobname'])
# setup AppPot parameters
use_apppot = False
apppot_img = None
apppot_changes = None
apppot_file = sessionParas['AppPotFile']
if apppot_file:
use_apppot = True
if apppot_file.endswith('.changes.tar.gz'):
apppot_changes = apppot_file
else:
apppot_img = apppot_file
pathToExecutable = sessionParas['pathToExecutable']
localBaseDir = sessionParas['localBaseDir']
architecture = sessionParas['architecture']
jobname = sessionParas['jobname']
executable = os.path.basename(self.pathToExecutable)
inputs = { self.pathToExecutable:executable }
# make a "stage" directory where input files are collected
path_to_stage_dir = os.path.join(self.paraFolder, jobname)
gc3libs.utils.mkdir(path_to_stage_dir)
prefix_len = len(path_to_stage_dir) + 1
# 2. apply substitutions to parameter files
for (path, changes) in substs.iteritems():
for (var, val, index, regex) in changes:
support.update_parameter_in_file(os.path.join(localBaseDir, path),
var, index, val, regex)
support.fillInputDir(localBaseDir, path_to_stage_dir)
# 3. build input file list
for dirpath,dirnames,filenames in os.walk(path_to_stage_dir):
for filename in filenames:
# cut the leading part, which is == to path_to_stage_dir
relpath = dirpath[prefix_len:]
# ignore output directory contents in resubmission
if relpath. startswith('output'):
continue
remote_path = os.path.join(relpath, filename)
inputs[os.path.join(dirpath, filename)] = remote_path
# all contents of the `output` directory are to be fetched
outputs = { 'output/':'' }
kwargs = {}
kwargs['stdout'] = 'idRisk.log'
kwargs['join'] = True
kwargs['output_dir'] = os.path.join(path_to_stage_dir, 'output')
kwargs['requested_architecture'] = self.architecture
# adaptions for uml
if use_apppot:
if apppot_img is not None:
kwargs['apppot_img'] = apppot_img
if apppot_changes is not None:
kwargs['apppot_changes'] = apppot_changes
cls = idRiskApppotApplication
else:
cls = idRiskApplication
kwargs.setdefault('tags', [ ])
# hand over job to create
# self.curApplication = cls('/home/user/job/' + executable, [], inputs, outputs, **kwargs)
self.curApplication = cls('./' + executable, [], inputs, outputs, **kwargs)
SequentialTaskCollection.__init__(self, self.jobname, [ self.curApplication ])
logger.debug('done gParaSearchDriver.__init__ for job %s' % sessionParas['jobname'])
def __init__(self, *args, **kwargs):
SequentialTaskCollection.__init__(self, *args, **kwargs)
self.next_called_n_times = 0
def __init__(self, num_tasks, **extra_args):
tasks = [
SuccessfulApp('stage{n}'.format(n=n)) for n in range(num_tasks)
]
SequentialTaskCollection.__init__(self, tasks, **extra_args)
def __init__(self, *args, **kwargs):
SequentialTaskCollection.__init__(self, *args, **kwargs)
self.next_called_n_times = 0
def __init__(self, pathToExecutable, pathToStageDir, architecture, baseDir, xVars, initialPop,
nPopulation, domain, solverVerb, problemType, pathEmpirical,
itermax, xConvCrit, yConvCrit,
makePlots, optStrategy, fWeight, fCritical, ctryList, analyzeResults, nlc, plot3dTable, combOverviews,
output_dir = '/tmp', **extra_args):
'''
pathToExecutable: Path to main executable.
pathToStageDir : Path to directory in which the action takes place. Usually set to os.getcwd().
architecture : Set the architecture used.
baseDir : Directory in which the input files are assembled. This directory is sent as input to the cluseter.
xVars : Names of the x variables.
initialPop : The initial population if recovering from earlier failure.
nPopulation : Population size.
domain : The domain of the x variables. List of (lowerbound, upperbound) tuples.
solverVerb : Verbosity of the solver.
problemType : The problem type of the forward premium optimization. One4all, one4each etc.
pathEmpirical : Path to the empirical results of the forward premium project.
itermax : Maximum # of iterations of the solver.
xConvCrit : Convergence criteria for x variables.
yConvCrit : Convergence criteria for the y variables.
makePlots : Make plots?
optStrategy : The kind of differential evolution strategy to use.
fWeight :
fCritical :
ctryList : The list of ctrys analyzed.
analyzeResults : Function to analyze the output retrieved from the servers.
nlc : Constraint function.
plot3dTable : Function to generate 3d plots.
combOverviews : Function to combine overviews.
'''
logger.debug('entering gParaSearchDriver.__init__')
# Set up initial variables and set the correct methods.
self.pathToStageDir = pathToStageDir
self.problemType = problemType
self.pathToExecutable = pathToExecutable
self.architecture = architecture
self.baseDir = baseDir
self.verbosity = solverVerb.upper()
self.jobname = extra_args['jobname']
self.ctryList = ctryList.split()
self.xVars = xVars
self.domain = domain
self.n = len(self.xVars.split())
self.extra_args = extra_args
self.analyzeResults = analyzeResults
self.plot3dTable = plot3dTable
self.combOverviews = combOverviews
# Set solver options
S_struct = {}
S_struct['I_NP'] = int(nPopulation)
S_struct['F_weight'] = float(fWeight)
S_struct['F_CR'] = float(fCritical)
S_struct['I_D'] = self.n
S_struct['lowerBds'] = np.array([ element[0] for element in domain ], dtype = 'float64')
S_struct['upperBds'] = np.array([ element[1] for element in domain ], dtype = 'float64')
S_struct['I_itermax'] = int(itermax)
S_struct['F_VTR'] = float(yConvCrit)
S_struct['I_strategy'] = int(optStrategy)
S_struct['I_plotting'] = int(makePlots)
S_struct['xConvCrit'] = float(xConvCrit)
S_struct['workingDir'] = pathToStageDir
S_struct['verbosity'] = self.verbosity
# Initialize solver
self.deSolver = deKenPrice(S_struct)
# Set constraint function in solver.
self.deSolver.nlc = nlc
# create initial task and register it
if initialPop:
self.deSolver.newPop = np.loadtxt(initialPop, delimiter = ' ')
else:
self.deSolver.newPop = self.deSolver.drawInitialSample()
#self.deSolver.plotPopulation(self.deSolver.newPop)
self.deSolver.I_iter += 1
self.evaluator = gParaSearchParallel(self.deSolver.newPop, self.deSolver.I_iter, self.pathToExecutable, self.pathToStageDir,
self.architecture, self.baseDir, self.xVars, self.verbosity, self.problemType, self.analyzeResults,
self.ctryList, **self.extra_args)
initial_task = self.evaluator
SequentialTaskCollection.__init__(self, self.jobname, [initial_task])
def __init__(self, single_run_duration, generations_to_do,
N, p_mut_coeff, choose_or_rand, sick_or_not, off_v_last,
output_dir, executable=None, **extra_args):
"""Create a new task running an ``MHC_coev`` binary.
Each binary is expected to run for `single_run_duration`
minutes and dump its state in file ``latest_work.mat`` if it's
not finished. This task will continue re-submitting the same
executable together with the saved workspace until
`generations_to_do` generations have been computed.
:param single_run_duration:
Duration of a single step (as a `gc3libs.quantity.Duration`:class: value)
:param int generations_to_do:
Count of generations that ``MHC_coev`` should simulate.
:param N: Passed unchanged to the MHC_coev program.
:param p_mut_coeff: Passed unchanged to the MHC_coev program.
:param choose_or_rand: Passed unchanged to the MHC_coev program.
:param sick_or_not: Passed unchanged to the MHC_coev program.
:param off_v_last: Passed unchanged to the MHC_coev program.
:param str output_dir:
Path to a directory where output files
from all runs should be collected.
:param str executable:
Path to the ``MHC_coev`` executable binary, or `None`
(default) to specify that the default version available on
the execution site should be used.
"""
# remember values for later use
self.executable = executable
self.output_dir = output_dir
# allow 5 extra minutes for final saving the MatLab workspace
self.single_run_duration = single_run_duration + 5*minutes
self.generations_to_do = generations_to_do
self.p_mut_coeff = p_mut_coeff
self.N = N
self.choose_or_rand = choose_or_rand
self.sick_or_not = sick_or_not
self.off_v_last = off_v_last
self.extra = extra_args
self.generations_done = 0
# make up a sensibel job name if there isn't one
if 'jobname' in extra_args:
self.jobname = extra_args['jobname']
else:
if self.executable is None:
self.jobname = ('MHC_coev.%s.%s.%s.%s.%s'
% (N, p_mut_coeff, choose_or_rand, sick_or_not, off_v_last))
else:
os.path.basename(self.executable)
# remove potentially conflicting kyword arguments
extra_args.pop('output_dir', None)
extra_args.pop('requested_walltime', None)
# create initial task and register it
initial_task = GMhcCoevApplication(
N, p_mut_coeff, choose_or_rand, sick_or_not, off_v_last,
output_dir = os.path.join(output_dir, 'tmp'),
executable = self.executable,
requested_walltime = self.single_run_duration,
**extra_args)
SequentialTaskCollection.__init__(self, [initial_task])
def __init__(self, pathToExecutable, pathToStageDir, architecture, baseDir, xVars, initialPop,
nPopulation, domain, solverVerb, problemType, pathEmpirical,
itermax, xConvCrit, yConvCrit,
makePlots, optStrategy, fWeight, fCritical, ctryList, analyzeResults, nlc, plot3dTable, combOverviews,
output_dir = '/tmp', **extra_args):
'''
pathToExecutable: Path to main executable.
pathToStageDir : Path to directory in which the action takes place. Usually set to os.getcwd().
architecture : Set the architecture used.
baseDir : Directory in which the input files are assembled. This directory is sent as input to the cluseter.
xVars : Names of the x variables.
initialPop : The initial population if recovering from earlier failure.
nPopulation : Population size.
domain : The domain of the x variables. List of (lowerbound, upperbound) tuples.
solverVerb : Verbosity of the solver.
problemType : The problem type of the forward premium optimization. One4all, one4each etc.
pathEmpirical : Path to the empirical results of the forward premium project.
itermax : Maximum # of iterations of the solver.
xConvCrit : Convergence criteria for x variables.
yConvCrit : Convergence criteria for the y variables.
makePlots : Make plots?
optStrategy : The kind of differential evolution strategy to use.
fWeight :
fCritical :
ctryList : The list of ctrys analyzed.
analyzeResults : Function to analyze the output retrieved from the servers.
nlc : Constraint function.
plot3dTable : Function to generate 3d plots.
combOverviews : Function to combine overviews.
'''
logger.debug('entering gParaSearchDriver.__init__')
# Set up initial variables and set the correct methods.
self.pathToStageDir = pathToStageDir
self.problemType = problemType
self.pathToExecutable = pathToExecutable
self.architecture = architecture
self.baseDir = baseDir
self.verbosity = solverVerb.upper()
self.jobname = extra_args['jobname']
self.ctryList = ctryList.split()
self.xVars = xVars
self.domain = domain
self.n = len(self.xVars.split())
self.extra_args = extra_args
self.analyzeResults = analyzeResults
self.plot3dTable = plot3dTable
self.combOverviews = combOverviews
# Set solver options
S_struct = {}
S_struct['I_NP'] = int(nPopulation)
S_struct['F_weight'] = float(fWeight)
S_struct['F_CR'] = float(fCritical)
S_struct['I_D'] = self.n
S_struct['lowerBds'] = np.array([ element[0] for element in domain ], dtype = 'float64')
S_struct['upperBds'] = np.array([ element[1] for element in domain ], dtype = 'float64')
S_struct['I_itermax'] = int(itermax)
S_struct['F_VTR'] = float(yConvCrit)
S_struct['I_strategy'] = int(optStrategy)
S_struct['I_plotting'] = int(makePlots)
S_struct['xConvCrit'] = float(xConvCrit)
S_struct['workingDir'] = pathToStageDir
S_struct['verbosity'] = self.verbosity
# Initialize solver
self.deSolver = deKenPrice(S_struct)
# Set constraint function in solver.
self.deSolver.nlc = nlc
# create initial task and register it
if initialPop:
self.deSolver.newPop = np.loadtxt(initialPop, delimiter = ' ')
else:
self.deSolver.newPop = self.deSolver.drawInitialSample()
#self.deSolver.plotPopulation(self.deSolver.newPop)
self.deSolver.I_iter += 1
self.evaluator = gParaSearchParallel(self.deSolver.newPop, self.deSolver.I_iter, self.pathToExecutable, self.pathToStageDir,
self.architecture, self.baseDir, self.xVars, self.verbosity, self.problemType, self.analyzeResults,
self.ctryList, **self.extra_args)
initial_task = self.evaluator
SequentialTaskCollection.__init__(self, self.jobname, [initial_task])
def __init__(self, num_tasks, **extra_args):
tasks = [SuccessfulApp('stage{n}'.format(n=n)) for n in range(num_tasks)]
SequentialTaskCollection.__init__(self, tasks, **extra_args)