def test_SequentialDriver_with_rosenbrock(self): """Test :class:`gc3libs.optimizer.drivers.SequentialDriver """ # Test parameters magic_seed = 100 dim = 2 pop_size = 100 lower_bounds = -2 * np.ones(dim) upper_bounds = +2 * np.ones(dim) prob_cross = 0.8 configure_logger(level=logging.CRITICAL) log = logging.getLogger("gc3.gc3libs") initial_pop = draw_population( lower_bds=lower_bounds, upper_bds=upper_bounds, dim=dim, size=pop_size, in_domain=self.rosenbrock_sample_filter, seed=magic_seed) algo = DifferentialEvolutionAlgorithm( initial_pop=initial_pop, de_step_size=0.85, # DE-stepsize ex [0, 2] prob_crossover=prob_cross, # crossover probabililty constant ex [0, 1] itermax=1000, # maximum number of iterations (generations) dx_conv_crit=None, # stop when variation among x's is < this y_conv_crit=1e-5, # stop when ofunc < y_conv_crit de_strategy='DE_local_to_best', logger=log, in_domain=self.rosenbrock_sample_filter, seed=magic_seed, after_update_opt_state=[print_stats, log_stats] #, plot_population(temp_stage_dir)] ) assert algo.de_step_size == 0.85 assert algo.prob_crossover == prob_cross assert algo.itermax == 1000 assert algo.dx_conv_crit is None assert algo.y_conv_crit == 1e-5 assert algo.de_strategy == 'DE_local_to_best' assert algo.logger == log opt = SequentialDriver( algo, target_fn=self.rosenbrock_fn, fmt="%12.8f") assert opt.target_fn == self.rosenbrock_fn # run the Diff.Evo. algorithm opt.de_opt() assert algo.has_converged() assert (algo.best_y - 0.) < algo.y_conv_crit assert (algo.best_x[0] - 1.) < 1e-3 assert (algo.best_x[1] - 1.) < 1e-3
def test_SequentialDriver_with_rosenbrock(self): """Test :class:`gc3libs.optimizer.drivers.SequentialDriver """ # Test parameters magic_seed = 100 dim = 2 pop_size = 100 lower_bounds = -2 * np.ones(dim) upper_bounds = +2 * np.ones(dim) prob_cross = 0.8 configure_logger(level=logging.CRITICAL) log = logging.getLogger("gc3.gc3libs") initial_pop = draw_population(lower_bds=lower_bounds, upper_bds=upper_bounds, dim=dim, size=pop_size, in_domain=self.rosenbrock_sample_filter, seed=magic_seed) algo = DifferentialEvolutionAlgorithm( initial_pop=initial_pop, de_step_size=0.85, # DE-stepsize ex [0, 2] prob_crossover=prob_cross, # crossover probabililty constant ex [0, 1] itermax=1000, # maximum number of iterations (generations) dx_conv_crit=None, # stop when variation among x's is < this y_conv_crit=1e-5, # stop when ofunc < y_conv_crit de_strategy='DE_local_to_best', logger=log, in_domain=self.rosenbrock_sample_filter, seed=magic_seed, after_update_opt_state=[print_stats, log_stats] #, plot_population(temp_stage_dir)] ) assert algo.de_step_size == 0.85 assert algo.prob_crossover == prob_cross assert algo.itermax == 1000 assert algo.dx_conv_crit is None assert algo.y_conv_crit == 1e-5 assert algo.de_strategy == 'DE_local_to_best' assert algo.logger == log opt = SequentialDriver(algo, target_fn=self.rosenbrock_fn, fmt="%12.8f") assert opt.target_fn == self.rosenbrock_fn # run the Diff.Evo. algorithm opt.de_opt() assert algo.has_converged() assert (algo.best_y - 0.) < algo.y_conv_crit assert (algo.best_x[0] - 1.) < 1e-3 assert (algo.best_x[1] - 1.) < 1e-3
def pre_run(self): """ Temporary overload for pre_run method of gc3libs.cmdline._Script. """ import cli # pyCLI import cli.app import cli._ext.argparse as argparse from cli.util import ifelse, ismethodof import logging ## finish setup self.setup_options() self.setup_args() ## parse command-line cli.app.CommandLineApp.pre_run(self) ## setup GC3Libs logging loglevel = max( 1, logging.ERROR - 10 * max(0, self.params.verbose - self.verbose_logging_threshold)) gc3libs.configure_logger(loglevel, self.name) self.log = logging.getLogger( 'gc3.gc3utils') # alternate: ('gc3.' + self.name) self.log.setLevel(loglevel) self.log.propagate = True self.log.parent.propagate = False # Changed to false since we want to avoid dealing with the root logger and catch the information directly. # temporarily take out logging redirection # # from logging import getLogger # # from logbook.compat import redirect_logging # from logbook.compat import RedirectLoggingHandler # # redirect_logging() # does the same thing as adding a RedirectLoggingHandler... might as well be explicit # self.log.parent.handlers = [] # self.log.parent.addHandler(RedirectLoggingHandler()) # print self.log.handlers # print self.log.parent.handlers # print self.log.root.handlers # self.log.critical('redirected gc3 log to ' + curFileName + '.log.') # interface to the GC3Libs main functionality self._core = self._get_core() # call hook methods from derived classes self.parse_args()
def pre_run(self): """ Temporary overload for pre_run method of gc3libs.cmdline._Script. """ import cli # pyCLI import cli.app import cli._ext.argparse as argparse from cli.util import ifelse, ismethodof import logging ## finish setup self.setup_options() self.setup_args() ## parse command-line cli.app.CommandLineApp.pre_run(self) ## setup GC3Libs logging loglevel = max(1, logging.ERROR - 10 * max(0, self.params.verbose - self.verbose_logging_threshold)) gc3libs.configure_logger(loglevel, self.name) self.log = logging.getLogger("gc3.gc3utils") # alternate: ('gc3.' + self.name) self.log.setLevel(loglevel) self.log.propagate = True self.log.parent.propagate = False # Changed to false since we want to avoid dealing with the root logger and catch the information directly. from logging import getLogger from logbook.compat import redirect_logging from logbook.compat import RedirectLoggingHandler # redirect_logging() # does the same thing as adding a RedirectLoggingHandler... might as well be explicit self.log.parent.handlers = [] self.log.parent.addHandler(RedirectLoggingHandler()) print self.log.handlers print self.log.parent.handlers print self.log.root.handlers self.log.critical("Successfully overridden gc3pie error handling. ") # interface to the GC3Libs main functionality self._core = self._make_core() # call hook methods from derived classes self.parse_args()
help="Increase program verbosity" " (default is 0; any higher number may spoil screen output).", ) cmdline.add_option("-w", "--wall-clock-time", dest="wctime", default=str(8), # 8 hrs metavar="DURATION", help="Each GAMESS job will run for at most DURATION time" " (default: %default hours), after which it" " will be killed and considered failed. DURATION can be a whole" " number, expressing duration in hours, or a string of the form HH:MM," " specifying that a job can last at most HH hours and MM minutes." ) (options, args) = cmdline.parse_args() # set up logging loglevel = max(1, logging.ERROR - 10 * options.verbose) gc3libs.configure_logger(loglevel, "smscg-run") logger = logging.getLogger("gc3.smscg-run") logger.setLevel(loglevel) logger.propagate = True # parse args exe = None if len(args) != 0: exe = args[0] del args[0] if not os.path.isabs(exe): exe = os.path.join(os.getcwd(), exe) if exe is not None and len(args) == 0: cmdline.error("Missing BATCH specification.") else:
# You should have received a copy of the GNU Lesser General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # __docformat__ = 'reStructuredText' import logging import os import shutil import tempfile from gc3libs import Application, Run, configure_logger, create_engine from gc3libs.workflow import SequentialTaskCollection loglevel = logging.ERROR configure_logger(loglevel, "test_issue_335") class MySequentialCollection(SequentialTaskCollection): def __init__(self, *args, **kwargs): SequentialTaskCollection.__init__(self, *args, **kwargs) self.next_called_n_times = 0 def next(self, x): """count times next() is called""" self.next_called_n_times += 1 return SequentialTaskCollection.next(self, x) class test_issue_335(object):
You can specify the resource you want to use by passing its name as command line argument. """ # stdlib imports import sys import time # GC3Pie imports import gc3libs import logging # loglevel = logging.DEBUG loglevel = logging.WARN gc3libs.configure_logger(loglevel, "gc3hucore") class HucoreDeconvolveApplication(gc3libs.Application): """ This application calls `hucore` with a given template file and retrives the stdout/stderr in a file named `stdout.txt` plus the directories `resultdir` and `previews` into a directory `deconvolved` inside the current directory. """ def __init__(self): gc3libs.Application.__init__( self, arguments = ["/usr/local/bin/hucore", '-template', 'hucore_template_relative.tcl'], inputs = ['./hucore_template_relative.tcl', 'bad.lsm'], outputs = ['resultdir', 'previews'], output_dir = './deconvolved',
__changelog__ = """ """ __docformat__ = 'reStructuredText' import os import sys from gc3libs import Application from gc3libs.cmdline import SessionBasedScript from gc3libs.optimizer.utils import update_parameter_in_file from gc3libs.application.gamess import GamessApplication import numpy as np import logging import gc3libs gc3libs.configure_logger(logging.DEBUG) # optimizer specific imports from gc3libs.optimizer.drivers import ParallelDriver from gc3libs.optimizer.dif_evolution import DifferentialEvolutionParallel optimization_dir = os.path.join(os.getcwd(), 'optimizeGeometry') float_fmt = '%25.15f' np.set_printoptions(linewidth=300, precision=8, suppress=True) def compute_target_geometries(pop_task_tuple): ''' Given a list of (population, task), compute and return list of target values.
import sys import time import os # GC3Pie imports import gc3libs import gc3libs.config import gc3libs.core # Configure logging. This is not really necessary but will avoid some # boring errors from the logging subsystem. Moreover, it's important # to set it at least at ERROR level, otherwise some errors from your # code could be silently ignored. loglevel = logging.DEBUG gc3libs.configure_logger(loglevel, "gdemo") class GdemoSimpleApp(gc3libs.Application): """This simple application will run /bin/hostname on the remove host, and retrive the output in a file named `stdout.txt` into a directory `mygc3job` inside the current directory.""" def __init__(self): gc3libs.Application.__init__( self, arguments = ['/bin/cat', '/proc/cpuinfo'], # mandatory inputs = [], # mandatory outputs = [], # mandatory output_dir = "./mygc3job", # mandatory stdout = "stdout.txt",) def terminated(self):
def drive_optimization( population_size, dim, lower_bounds, upper_bounds, # DE-specific parameters de_strategy="DE_local_to_best", de_step_size=0.85, prob_crossover=1.0, exp_cross=False, # converge-related parameters itermax=100, dx_conv_crit=1e-6, y_conv_crit=None, # misc in_domain=None, seed=None, logger=None, after_update_opt_state=[], ): """ Driver script It uses DifferentialEvolutionAlgorithm as an implementation of Ken Price's differential evolution algorithm: [[http://www1.icsi.berkeley.edu/~storn/code.html]]. """ if logger is None: configure_logger(level=logging.CRITICAL) logger = logging.getLogger("gc3.gc3libs") if in_domain is None: def _default_in_domain(pop): return (pop < upper_bounds).all(axis=1) & (pop > lower_bounds).all(axis=1) in_domain = _default_in_domain opt = DifferentialEvolutionAlgorithm( initial_pop=np.zeros((population_size, dim)), de_step_size=de_step_size, # DE-stepsize ex [0, 2] prob_crossover=prob_crossover, # crossover probabililty constant ex [0, 1] itermax=itermax, # maximum number of iterations (generations) dx_conv_crit=dx_conv_crit, # stop when variation among x's is < this y_conv_crit=y_conv_crit, # stop when ofunc < y_conv_crit de_strategy=de_strategy, logger=logger, in_domain=in_domain, ) opt.vals = np.ones(population_size) * PENALTY_VALUE # init """ Jobs: create and manage population """ try: pop = getJobs(throw=True) except Exception as ex: # server error print ex return if not pop: # empty # Initialize population using the arguments passed to the # DifferentialEvolutionParallel initialization opt.new_pop = draw_population(lower_bds=lower_bounds, upper_bds=upper_bounds, size=population_size, dim=dim) putJobs(pop2Jobs(opt)) else: # finished? finished, count = True, 0 for job in pop: finished &= job.finished count += job.finished cur_iter = job.iteration - 1 # opt iter index start with 0 print "Iter(%d): %d finished jobs" % (cur_iter + 1, count) if opt.cur_iter != cur_iter: restoreCurrentPop(opt, cur_iter) # restore current population and iteration counter if finished: # Update population and evaluate convergence newVals = np.zeros(population_size) opt.new_pop = np.zeros((population_size, dim)) k = 0 for job in pop: opt.new_pop[k, :] = job.params newVals[k] = job.result if job.result != None else PENALTY_VALUE k += 1 opt.update_opt_state(opt.new_pop, newVals) putPop(opt) print [opt.best_y, opt.best_x] if opt.cur_iter > opt.itermax: print "Maximum number of iterations exceeded after [%d] steps. " % (opt.cur_iter) # sys.exit() if not opt.has_converged(): # Generate new population and enforce constrains opt.new_pop = opt.evolve() # Push all and run again! putJobs(pop2Jobs(opt)) return True else: # Once iteration has terminated, extract `bestval` which should represent # the element in *all* populations that lead to the closest match to the # empirical value print "Calibration converged after [%d] steps. " % (opt.cur_iter) sys.exit() # # VM's: create and manage dispatchers # vms = getVMs() # # if not vms: # empty # print "[+] No running EC2 instances found, creating %d" % N_NODES # nodes = fp_ec2_create_vms(N_NODES, pubkey_file='/home/tklauser/.ssh/id_rsa.pub') # vms = [] # for node in nodes: # vm = { 'ip' : node.public_ips[0], 'vmtype' : 'Amazon', 'dateUpdate' : str(datetime.datetime.now()) } # vms.append(vm) # putVMs(vms) # else: # pass #TODO manage VMs # Then, we could also run the forwardPremium binary here; Single script solution return False
def pre_run(self): ### FIXME: Some code copied from _Script.pre_run() self.setup_options() self.setup_args() cli.app.CommandLineApp.pre_run(self) loglevel = max(1, logging.WARNING - 10 * max(0, self.params.verbose - self.verbose_logging_threshold)) gc3libs.configure_logger(loglevel, "gc3.gc3utils") # alternate: self.name logging.root.setLevel(loglevel) # alternate: ('gc3.' + self.name) self.log = logging.getLogger('gc3.gc3utils') self.log.setLevel(loglevel) self.log.propagate = True self.log.info("Starting %s at %s; invoked as '%s'", self.name, time.asctime(), str.join(' ', sys.argv)) # FIXME: we need to ignore the process_args method as the # `client` subparser has different options than the `server` # one, and the default is the `server` subparser. if self.params.func == self._main_client: # override `process_args` with a noop. self.process_args = lambda *x, **kw: None return # Read config file(s) from command line self.params.config_files = self.params.config_files.split(',') # interface to the GC3Libs main functionality self.config = self._make_config(self.params.config_files) try: self._core = gc3libs.core.Core(self.config) except gc3libs.exceptions.NoResources: # translate internal error `NoResources` to a # user-readable message. raise gc3libs.exceptions.FatalError( "No computational resources defined." " Please edit the configuration file(s): '%s'." % (str.join("', '", self.params.config_files))) self.params.working_dir = os.path.abspath(self.params.working_dir) # Default session dir is inside the working directory if not self.params.session: self.params.session = os.path.join( self.params.working_dir, self.name) # Convert inbox to Url objects self.params.inbox = [gc3libs.url.Url(i) for i in self.params.inbox] # Default output directory is the working directory. if not self.params.output: self.params.output = self.params.working_dir self._prerun_common_checks() self.parse_args() # Syntax check for notify events self.params.notify_state = self.params.notify_state.split(',') # Add IN_ as we use shorter names for the command line state_names = ['IN_' + i for i in self.params.notify_state] # Build the notify mask, for later use self.notify_event_mask = 0 # Ensure all the supplied states are correct for istate in state_names: if istate not in notify_events: raise gc3libs.exceptions.InvalidUsage( "Invalid notify state %s." % state) self.notify_event_mask |= notify_events[istate]
""" __docformat__ = 'reStructuredText' import os import sys from gc3libs import Application from gc3libs.cmdline import SessionBasedScript from gc3libs.optimizer.utils import update_parameter_in_file from gc3libs.application.gamess import GamessApplication import numpy as np import logging import gc3libs gc3libs.configure_logger(logging.DEBUG) # optimizer specific imports from gc3libs.optimizer.drivers import ParallelDriver from gc3libs.optimizer.dif_evolution import DifferentialEvolutionParallel optimization_dir = os.path.join(os.getcwd(), 'optimizeGeometry') float_fmt = '%25.15f' np.set_printoptions(linewidth = 300, precision = 8, suppress = True) def compute_target_geometries(pop_task_tuple): ''' Given a list of (population, task), compute and return list of target
import gc3libs gc3libs.configure_logger(30) gc3conffile = 'config/samples/gc3pie_localhost.conf' engine = gc3libs.create_engine(gc3conffile) core = engine._core
import sys import time import gc3libs import pyinotify import pprint import HRM import logging loglevel = logging.WARN gc3libs.configure_logger(loglevel, "qmgc3") logw = gc3libs.log.warn logi = gc3libs.log.info logd = gc3libs.log.debug fname = '/var/www/hrm/run/spool/new/deconvolution_job.cfg' job = HRM.JobDescription(fname, 'file')
# from __future__ import absolute_import, print_function, unicode_literals from builtins import range from builtins import object __docformat__ = 'reStructuredText' import logging import os import shutil import tempfile from gc3libs import Application, Run, configure_logger, create_engine from gc3libs.workflow import SequentialTaskCollection loglevel = logging.ERROR configure_logger(loglevel, "test_issue_335") class MySequentialCollection(SequentialTaskCollection): def __init__(self, *args, **kwargs): SequentialTaskCollection.__init__(self, *args, **kwargs) self.next_called_n_times = 0 def next(self, x): """count times next() is called""" self.next_called_n_times += 1 return SequentialTaskCollection.next(self, x) class test_issue_335(object): def setUp(self):
""" # stdlib imports import logging import sys import time # GC3Pie imports import gc3libs import gc3libs.config import gc3libs.core # Configure logging. Without this, you won't see any messages from GC3Pie. # Possible levels are: DEBUG, INFO, WARNING, ERROR loglevel = logging.DEBUG gc3libs.configure_logger(loglevel, "gdemo") class SquareApplication(gc3libs.Application): """Compute the square of `x`, using a remote job.""" def __init__(self, x): self.to_square = x gc3libs.Application.__init__(self, arguments=['expr', x, '*', x], inputs=[], outputs=[], output_dir="./squares.d", stdout="stdout.txt", join=True)
You can specify the resource you want to use by passing its name as command line argument. """ # stdlib imports import sys import time # GC3Pie imports import gc3libs import logging loglevel = logging.DEBUG # loglevel = logging.WARN gc3libs.configure_logger(loglevel, "lsbrelease") class LSBReleaseApp(gc3libs.Application): """ This simple application will call `lsb_release -a` and retrive the output in a file named `stdout.txt` into a directory `lsbrelease` inside the current directory. """ def __init__(self): gc3libs.Application.__init__( self, arguments = ["/usr/bin/lsb_release", '-a'], # mandatory inputs = [], # mandatory outputs = [], # mandatory output_dir = "./lsbrelease", # mandatory stderr = 'stdout.txt', # combine stdout & stderr