def test_SequentialDriver_with_rosenbrock(self):
"""Test :class:`gc3libs.optimizer.drivers.SequentialDriver
"""
# Test parameters
magic_seed = 100
dim = 2
pop_size = 100
lower_bounds = -2 * np.ones(dim)
upper_bounds = +2 * np.ones(dim)
prob_cross = 0.8
configure_logger(level=logging.CRITICAL)
log = logging.getLogger("gc3.gc3libs")
initial_pop = draw_population(
lower_bds=lower_bounds,
upper_bds=upper_bounds,
dim=dim,
size=pop_size,
in_domain=self.rosenbrock_sample_filter,
seed=magic_seed)
algo = DifferentialEvolutionAlgorithm(
initial_pop=initial_pop,
de_step_size=0.85, # DE-stepsize ex [0, 2]
prob_crossover=prob_cross,
# crossover probabililty constant ex [0, 1]
itermax=1000, # maximum number of iterations (generations)
dx_conv_crit=None, # stop when variation among x's is < this
y_conv_crit=1e-5, # stop when ofunc < y_conv_crit
de_strategy='DE_local_to_best',
logger=log,
in_domain=self.rosenbrock_sample_filter,
seed=magic_seed,
after_update_opt_state=[print_stats, log_stats]
#, plot_population(temp_stage_dir)]
)
assert algo.de_step_size == 0.85
assert algo.prob_crossover == prob_cross
assert algo.itermax == 1000
assert algo.dx_conv_crit is None
assert algo.y_conv_crit == 1e-5
assert algo.de_strategy == 'DE_local_to_best'
assert algo.logger == log
opt = SequentialDriver(
algo,
target_fn=self.rosenbrock_fn,
fmt="%12.8f")
assert opt.target_fn == self.rosenbrock_fn
# run the Diff.Evo. algorithm
opt.de_opt()
assert algo.has_converged()
assert (algo.best_y - 0.) < algo.y_conv_crit
assert (algo.best_x[0] - 1.) < 1e-3
assert (algo.best_x[1] - 1.) < 1e-3
def test_SequentialDriver_with_rosenbrock(self):
"""Test :class:`gc3libs.optimizer.drivers.SequentialDriver
"""
# Test parameters
magic_seed = 100
dim = 2
pop_size = 100
lower_bounds = -2 * np.ones(dim)
upper_bounds = +2 * np.ones(dim)
prob_cross = 0.8
configure_logger(level=logging.CRITICAL)
log = logging.getLogger("gc3.gc3libs")
initial_pop = draw_population(lower_bds=lower_bounds,
upper_bds=upper_bounds,
dim=dim,
size=pop_size,
in_domain=self.rosenbrock_sample_filter,
seed=magic_seed)
algo = DifferentialEvolutionAlgorithm(
initial_pop=initial_pop,
de_step_size=0.85, # DE-stepsize ex [0, 2]
prob_crossover=prob_cross,
# crossover probabililty constant ex [0, 1]
itermax=1000, # maximum number of iterations (generations)
dx_conv_crit=None, # stop when variation among x's is < this
y_conv_crit=1e-5, # stop when ofunc < y_conv_crit
de_strategy='DE_local_to_best',
logger=log,
in_domain=self.rosenbrock_sample_filter,
seed=magic_seed,
after_update_opt_state=[print_stats, log_stats]
#, plot_population(temp_stage_dir)]
)
assert algo.de_step_size == 0.85
assert algo.prob_crossover == prob_cross
assert algo.itermax == 1000
assert algo.dx_conv_crit is None
assert algo.y_conv_crit == 1e-5
assert algo.de_strategy == 'DE_local_to_best'
assert algo.logger == log
opt = SequentialDriver(algo,
target_fn=self.rosenbrock_fn,
fmt="%12.8f")
assert opt.target_fn == self.rosenbrock_fn
# run the Diff.Evo. algorithm
opt.de_opt()
assert algo.has_converged()
assert (algo.best_y - 0.) < algo.y_conv_crit
assert (algo.best_x[0] - 1.) < 1e-3
assert (algo.best_x[1] - 1.) < 1e-3
def pre_run(self):
"""
Temporary overload for pre_run method of gc3libs.cmdline._Script.
"""
import cli # pyCLI
import cli.app
import cli._ext.argparse as argparse
from cli.util import ifelse, ismethodof
import logging
## finish setup
self.setup_options()
self.setup_args()
## parse command-line
cli.app.CommandLineApp.pre_run(self)
## setup GC3Libs logging
loglevel = max(
1, logging.ERROR -
10 * max(0, self.params.verbose - self.verbose_logging_threshold))
gc3libs.configure_logger(loglevel, self.name)
self.log = logging.getLogger(
'gc3.gc3utils') # alternate: ('gc3.' + self.name)
self.log.setLevel(loglevel)
self.log.propagate = True
self.log.parent.propagate = False
# Changed to false since we want to avoid dealing with the root logger and catch the information directly.
# temporarily take out logging redirection
# # from logging import getLogger
# # from logbook.compat import redirect_logging
# from logbook.compat import RedirectLoggingHandler
# # redirect_logging() # does the same thing as adding a RedirectLoggingHandler... might as well be explicit
# self.log.parent.handlers = []
# self.log.parent.addHandler(RedirectLoggingHandler())
# print self.log.handlers
# print self.log.parent.handlers
# print self.log.root.handlers
# self.log.critical('redirected gc3 log to ' + curFileName + '.log.')
# interface to the GC3Libs main functionality
self._core = self._get_core()
# call hook methods from derived classes
self.parse_args()
def pre_run(self):
"""
Temporary overload for pre_run method of gc3libs.cmdline._Script.
"""
import cli # pyCLI
import cli.app
import cli._ext.argparse as argparse
from cli.util import ifelse, ismethodof
import logging
## finish setup
self.setup_options()
self.setup_args()
## parse command-line
cli.app.CommandLineApp.pre_run(self)
## setup GC3Libs logging
loglevel = max(1, logging.ERROR - 10 * max(0, self.params.verbose - self.verbose_logging_threshold))
gc3libs.configure_logger(loglevel, self.name)
self.log = logging.getLogger("gc3.gc3utils") # alternate: ('gc3.' + self.name)
self.log.setLevel(loglevel)
self.log.propagate = True
self.log.parent.propagate = False
# Changed to false since we want to avoid dealing with the root logger and catch the information directly.
from logging import getLogger
from logbook.compat import redirect_logging
from logbook.compat import RedirectLoggingHandler
# redirect_logging() # does the same thing as adding a RedirectLoggingHandler... might as well be explicit
self.log.parent.handlers = []
self.log.parent.addHandler(RedirectLoggingHandler())
print self.log.handlers
print self.log.parent.handlers
print self.log.root.handlers
self.log.critical("Successfully overridden gc3pie error handling. ")
# interface to the GC3Libs main functionality
self._core = self._make_core()
# call hook methods from derived classes
self.parse_args()
help="Increase program verbosity"
" (default is 0; any higher number may spoil screen output).",
)
cmdline.add_option("-w", "--wall-clock-time", dest="wctime", default=str(8), # 8 hrs
metavar="DURATION",
help="Each GAMESS job will run for at most DURATION time"
" (default: %default hours), after which it"
" will be killed and considered failed. DURATION can be a whole"
" number, expressing duration in hours, or a string of the form HH:MM,"
" specifying that a job can last at most HH hours and MM minutes."
)
(options, args) = cmdline.parse_args()
# set up logging
loglevel = max(1, logging.ERROR - 10 * options.verbose)
gc3libs.configure_logger(loglevel, "smscg-run")
logger = logging.getLogger("gc3.smscg-run")
logger.setLevel(loglevel)
logger.propagate = True
# parse args
exe = None
if len(args) != 0:
exe = args[0]
del args[0]
if not os.path.isabs(exe):
exe = os.path.join(os.getcwd(), exe)
if exe is not None and len(args) == 0:
cmdline.error("Missing BATCH specification.")
else:
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
__docformat__ = 'reStructuredText'
import logging
import os
import shutil
import tempfile
from gc3libs import Application, Run, configure_logger, create_engine
from gc3libs.workflow import SequentialTaskCollection
loglevel = logging.ERROR
configure_logger(loglevel, "test_issue_335")
class MySequentialCollection(SequentialTaskCollection):
def __init__(self, *args, **kwargs):
SequentialTaskCollection.__init__(self, *args, **kwargs)
self.next_called_n_times = 0
def next(self, x):
"""count times next() is called"""
self.next_called_n_times += 1
return SequentialTaskCollection.next(self, x)
class test_issue_335(object):
You can specify the resource you want to use by passing its name as
command line argument.
"""
# stdlib imports
import sys
import time
# GC3Pie imports
import gc3libs
import logging
# loglevel = logging.DEBUG
loglevel = logging.WARN
gc3libs.configure_logger(loglevel, "gc3hucore")
class HucoreDeconvolveApplication(gc3libs.Application):
"""
This application calls `hucore` with a given template file and retrives the
stdout/stderr in a file named `stdout.txt` plus the directories `resultdir`
and `previews` into a directory `deconvolved` inside the current directory.
"""
def __init__(self):
gc3libs.Application.__init__(
self,
arguments = ["/usr/local/bin/hucore",
'-template', 'hucore_template_relative.tcl'],
inputs = ['./hucore_template_relative.tcl', 'bad.lsm'],
outputs = ['resultdir', 'previews'],
output_dir = './deconvolved',
__changelog__ = """
"""
__docformat__ = 'reStructuredText'
import os
import sys
from gc3libs import Application
from gc3libs.cmdline import SessionBasedScript
from gc3libs.optimizer.utils import update_parameter_in_file
from gc3libs.application.gamess import GamessApplication
import numpy as np
import logging
import gc3libs
gc3libs.configure_logger(logging.DEBUG)
# optimizer specific imports
from gc3libs.optimizer.drivers import ParallelDriver
from gc3libs.optimizer.dif_evolution import DifferentialEvolutionParallel
optimization_dir = os.path.join(os.getcwd(), 'optimizeGeometry')
float_fmt = '%25.15f'
np.set_printoptions(linewidth=300, precision=8, suppress=True)
def compute_target_geometries(pop_task_tuple):
'''
Given a list of (population, task), compute and return list of target
values.
import sys
import time
import os
# GC3Pie imports
import gc3libs
import gc3libs.config
import gc3libs.core
# Configure logging. This is not really necessary but will avoid some
# boring errors from the logging subsystem. Moreover, it's important
# to set it at least at ERROR level, otherwise some errors from your
# code could be silently ignored.
loglevel = logging.DEBUG
gc3libs.configure_logger(loglevel, "gdemo")
class GdemoSimpleApp(gc3libs.Application):
"""This simple application will run /bin/hostname on the remove
host, and retrive the output in a file named `stdout.txt` into a
directory `mygc3job` inside the current directory."""
def __init__(self):
gc3libs.Application.__init__(
self,
arguments = ['/bin/cat', '/proc/cpuinfo'], # mandatory
inputs = [], # mandatory
outputs = [], # mandatory
output_dir = "./mygc3job", # mandatory
stdout = "stdout.txt",)
def terminated(self):
def drive_optimization(
population_size,
dim,
lower_bounds,
upper_bounds,
# DE-specific parameters
de_strategy="DE_local_to_best",
de_step_size=0.85,
prob_crossover=1.0,
exp_cross=False,
# converge-related parameters
itermax=100,
dx_conv_crit=1e-6,
y_conv_crit=None,
# misc
in_domain=None,
seed=None,
logger=None,
after_update_opt_state=[],
):
"""
Driver script
It uses DifferentialEvolutionAlgorithm as an implementation of
Ken Price's differential evolution
algorithm: [[http://www1.icsi.berkeley.edu/~storn/code.html]].
"""
if logger is None:
configure_logger(level=logging.CRITICAL)
logger = logging.getLogger("gc3.gc3libs")
if in_domain is None:
def _default_in_domain(pop):
return (pop < upper_bounds).all(axis=1) & (pop > lower_bounds).all(axis=1)
in_domain = _default_in_domain
opt = DifferentialEvolutionAlgorithm(
initial_pop=np.zeros((population_size, dim)),
de_step_size=de_step_size, # DE-stepsize ex [0, 2]
prob_crossover=prob_crossover, # crossover probabililty constant ex [0, 1]
itermax=itermax, # maximum number of iterations (generations)
dx_conv_crit=dx_conv_crit, # stop when variation among x's is < this
y_conv_crit=y_conv_crit, # stop when ofunc < y_conv_crit
de_strategy=de_strategy,
logger=logger,
in_domain=in_domain,
)
opt.vals = np.ones(population_size) * PENALTY_VALUE # init
""" Jobs: create and manage population """
try:
pop = getJobs(throw=True)
except Exception as ex: # server error
print ex
return
if not pop: # empty
# Initialize population using the arguments passed to the
# DifferentialEvolutionParallel initialization
opt.new_pop = draw_population(lower_bds=lower_bounds, upper_bds=upper_bounds, size=population_size, dim=dim)
putJobs(pop2Jobs(opt))
else:
# finished?
finished, count = True, 0
for job in pop:
finished &= job.finished
count += job.finished
cur_iter = job.iteration - 1 # opt iter index start with 0
print "Iter(%d): %d finished jobs" % (cur_iter + 1, count)
if opt.cur_iter != cur_iter:
restoreCurrentPop(opt, cur_iter) # restore current population and iteration counter
if finished:
# Update population and evaluate convergence
newVals = np.zeros(population_size)
opt.new_pop = np.zeros((population_size, dim))
k = 0
for job in pop:
opt.new_pop[k, :] = job.params
newVals[k] = job.result if job.result != None else PENALTY_VALUE
k += 1
opt.update_opt_state(opt.new_pop, newVals)
putPop(opt)
print [opt.best_y, opt.best_x]
if opt.cur_iter > opt.itermax:
print "Maximum number of iterations exceeded after [%d] steps. " % (opt.cur_iter)
# sys.exit()
if not opt.has_converged():
# Generate new population and enforce constrains
opt.new_pop = opt.evolve()
# Push all and run again!
putJobs(pop2Jobs(opt))
return True
else:
# Once iteration has terminated, extract `bestval` which should represent
# the element in *all* populations that lead to the closest match to the
# empirical value
print "Calibration converged after [%d] steps. " % (opt.cur_iter)
sys.exit()
# # VM's: create and manage dispatchers
# vms = getVMs()
#
# if not vms: # empty
# print "[+] No running EC2 instances found, creating %d" % N_NODES
# nodes = fp_ec2_create_vms(N_NODES, pubkey_file='/home/tklauser/.ssh/id_rsa.pub')
# vms = []
# for node in nodes:
# vm = { 'ip' : node.public_ips[0], 'vmtype' : 'Amazon', 'dateUpdate' : str(datetime.datetime.now()) }
# vms.append(vm)
# putVMs(vms)
# else:
# pass #TODO manage VMs
# Then, we could also run the forwardPremium binary here; Single script solution
return False
def pre_run(self):
### FIXME: Some code copied from _Script.pre_run()
self.setup_options()
self.setup_args()
cli.app.CommandLineApp.pre_run(self)
loglevel = max(1, logging.WARNING -
10 *
max(0, self.params.verbose -
self.verbose_logging_threshold))
gc3libs.configure_logger(loglevel, "gc3.gc3utils") # alternate: self.name
logging.root.setLevel(loglevel)
# alternate: ('gc3.' + self.name)
self.log = logging.getLogger('gc3.gc3utils')
self.log.setLevel(loglevel)
self.log.propagate = True
self.log.info("Starting %s at %s; invoked as '%s'",
self.name, time.asctime(), str.join(' ', sys.argv))
# FIXME: we need to ignore the process_args method as the
# `client` subparser has different options than the `server`
# one, and the default is the `server` subparser.
if self.params.func == self._main_client:
# override `process_args` with a noop.
self.process_args = lambda *x, **kw: None
return
# Read config file(s) from command line
self.params.config_files = self.params.config_files.split(',')
# interface to the GC3Libs main functionality
self.config = self._make_config(self.params.config_files)
try:
self._core = gc3libs.core.Core(self.config)
except gc3libs.exceptions.NoResources:
# translate internal error `NoResources` to a
# user-readable message.
raise gc3libs.exceptions.FatalError(
"No computational resources defined."
" Please edit the configuration file(s): '%s'."
% (str.join("', '", self.params.config_files)))
self.params.working_dir = os.path.abspath(self.params.working_dir)
# Default session dir is inside the working directory
if not self.params.session:
self.params.session = os.path.join(
self.params.working_dir, self.name)
# Convert inbox to Url objects
self.params.inbox = [gc3libs.url.Url(i) for i in self.params.inbox]
# Default output directory is the working directory.
if not self.params.output:
self.params.output = self.params.working_dir
self._prerun_common_checks()
self.parse_args()
# Syntax check for notify events
self.params.notify_state = self.params.notify_state.split(',')
# Add IN_ as we use shorter names for the command line
state_names = ['IN_' + i for i in self.params.notify_state]
# Build the notify mask, for later use
self.notify_event_mask = 0
# Ensure all the supplied states are correct
for istate in state_names:
if istate not in notify_events:
raise gc3libs.exceptions.InvalidUsage(
"Invalid notify state %s." % state)
self.notify_event_mask |= notify_events[istate]
"""
__docformat__ = 'reStructuredText'
import os
import sys
from gc3libs import Application
from gc3libs.cmdline import SessionBasedScript
from gc3libs.optimizer.utils import update_parameter_in_file
from gc3libs.application.gamess import GamessApplication
import numpy as np
import logging
import gc3libs
gc3libs.configure_logger(logging.DEBUG)
# optimizer specific imports
from gc3libs.optimizer.drivers import ParallelDriver
from gc3libs.optimizer.dif_evolution import DifferentialEvolutionParallel
optimization_dir = os.path.join(os.getcwd(), 'optimizeGeometry')
float_fmt = '%25.15f'
np.set_printoptions(linewidth = 300, precision = 8, suppress = True)
def compute_target_geometries(pop_task_tuple):
'''
Given a list of (population, task), compute and return list of target
import gc3libs gc3libs.configure_logger(30) gc3conffile = 'config/samples/gc3pie_localhost.conf' engine = gc3libs.create_engine(gc3conffile) core = engine._core
import sys import time import gc3libs import pyinotify import pprint import HRM import logging loglevel = logging.WARN gc3libs.configure_logger(loglevel, "qmgc3") logw = gc3libs.log.warn logi = gc3libs.log.info logd = gc3libs.log.debug fname = '/var/www/hrm/run/spool/new/deconvolution_job.cfg' job = HRM.JobDescription(fname, 'file')
#
from __future__ import absolute_import, print_function, unicode_literals
from builtins import range
from builtins import object
__docformat__ = 'reStructuredText'
import logging
import os
import shutil
import tempfile
from gc3libs import Application, Run, configure_logger, create_engine
from gc3libs.workflow import SequentialTaskCollection
loglevel = logging.ERROR
configure_logger(loglevel, "test_issue_335")
class MySequentialCollection(SequentialTaskCollection):
def __init__(self, *args, **kwargs):
SequentialTaskCollection.__init__(self, *args, **kwargs)
self.next_called_n_times = 0
def next(self, x):
"""count times next() is called"""
self.next_called_n_times += 1
return SequentialTaskCollection.next(self, x)
class test_issue_335(object):
def setUp(self):
"""
# stdlib imports
import logging
import sys
import time
# GC3Pie imports
import gc3libs
import gc3libs.config
import gc3libs.core
# Configure logging. Without this, you won't see any messages from GC3Pie.
# Possible levels are: DEBUG, INFO, WARNING, ERROR
loglevel = logging.DEBUG
gc3libs.configure_logger(loglevel, "gdemo")
class SquareApplication(gc3libs.Application):
"""Compute the square of `x`, using a remote job."""
def __init__(self, x):
self.to_square = x
gc3libs.Application.__init__(self,
arguments=['expr', x, '*', x],
inputs=[],
outputs=[],
output_dir="./squares.d",
stdout="stdout.txt",
join=True)
You can specify the resource you want to use by passing its name as command
line argument.
"""
# stdlib imports
import sys
import time
# GC3Pie imports
import gc3libs
import logging
loglevel = logging.DEBUG
# loglevel = logging.WARN
gc3libs.configure_logger(loglevel, "lsbrelease")
class LSBReleaseApp(gc3libs.Application):
"""
This simple application will call `lsb_release -a`
and retrive the output in a file named `stdout.txt` into a
directory `lsbrelease` inside the current directory.
"""
def __init__(self):
gc3libs.Application.__init__(
self,
arguments = ["/usr/bin/lsb_release", '-a'], # mandatory
inputs = [], # mandatory
outputs = [], # mandatory
output_dir = "./lsbrelease", # mandatory
stderr = 'stdout.txt', # combine stdout & stderr
