def __enter_class_context_managers(self, fixture_methods, callback):
"""Transform each fixture_method into a context manager with contextlib.contextmanager, enter them recursively, and call callback"""
if fixture_methods:
fixture_method = fixture_methods[0]
ctm = contextmanager(fixture_method)()
enter_result = TestResult(fixture_method)
enter_result.start()
self.fire_event(self.EVENT_ON_RUN_CLASS_SETUP_METHOD, enter_result)
if self.__execute_block_recording_exceptions(ctm.__enter__, enter_result, is_class_level=True):
enter_result.end_in_success()
self.fire_event(self.EVENT_ON_COMPLETE_CLASS_SETUP_METHOD, enter_result)
self.__enter_context_managers(fixture_methods[1:], callback)
exit_result = TestResult(fixture_method)
exit_result.start()
self.fire_event(self.EVENT_ON_RUN_CLASS_TEARDOWN_METHOD, exit_result)
if self.__execute_block_recording_exceptions(
lambda: ctm.__exit__(None, None, None), exit_result, is_class_level=True
):
exit_result.end_in_success()
self.fire_event(self.EVENT_ON_COMPLETE_CLASS_TEARDOWN_METHOD, exit_result)
else:
callback()
def test_request_context():
with contextmanager(appfactory)() as app:
templates = []
def capture(sender, template, context):
templates.append((template, context))
@jinja_rendered.connect_via(app)
def signal_jinja(sender, template, context):
template_rendered.send(None, template=template.name,
context=context)
try:
from flaskext.genshi import template_generated
except ImportError:
pass
else:
@template_generated.connect_via(app)
def signal_genshi(sender, template, context):
template_rendered.send(None, template=template.filename,
context=context)
with app_context(app) as client:
with template_rendered.connected_to(capture):
yield client, templates
def __enter_context_managers(self, fixture_methods, callback):
"""Transform each fixture_method into a context manager with contextlib.contextmanager, enter them recursively, and call callback"""
if fixture_methods:
with contextmanager(fixture_methods[0])():
self.__enter_context_managers(fixture_methods[1:], callback)
else:
callback()
def get_evaluation_context_getter():
if K.backend() == 'tensorflow':
import tensorflow as tf
return tf.get_default_graph().as_default
if K.backend() == 'theano':
return contextmanager(lambda: (yield))
def control_handler(f):
def g(*args):
with manage('allocator', 'var_manager', 'code_maker'):
yield
output = get_context().code_maker.output
f(output, *args)
return contextmanager(g)
def _context(self, *contexts, **kwargs):
with ExitStack() as stack:
res = []
for ctxname in contexts:
ctx = contextmanager(getattr(self, '_context_%s' % ctxname))
res.append(stack.enter_context(ctx(**kwargs)))
yield res if len(res) > 1 else res[0]
def assert_redirects():
manager = contextmanager(TestContext())
with manager() as client:
response = client.get("/bouncer/")
# First Django's API of passing the URL in by itself should work.
redirects(response, "https://example.com:1234/foo/?a=b#bar")
valids = [
{"port": 1234},
{"scheme": "https"},
{"domain": "example.com"},
{"query": "a=b"},
{"path": "/foo/"},
{"fragment": "bar"},
{"url": "https://example.com:1234/foo/?a=b#bar"},
]
for valid in valids:
redirects(response, **valid)
invalids = [
{"port": 5678},
{"scheme": "http"},
{"domain": "example.net"},
{"query": "c=d"},
{"path": "/baz/"},
{"fragment": "ben"},
{"url": "http://example.net:5678/baz/?c=d#ben"},
]
for invalid in invalids:
with attest.raises(AssertionError):
redirects(response, **invalid)
def g_decorator(generator_expr):
"""
Converts generator expression into a decorator
Takes in a generator expression, such as one accepted by
contextlib.contextmanager, converts it to a context manager,
and returns a decorator equivalent to being within that
context manager.
TODO do something with yielded value
Example:
@g_decorator
def foo():
print("Hello")
yield
print("World")
@foo()
def bar():
print("Something")
"""
cm = contextmanager(generator_expr)
@to_decorator
def wrapped_func(func, args, kwargs, *outer_args, **outer_kwargs):
with cm(*outer_args, **outer_kwargs) as yielded:
return func(*args, **kwargs)
return wrapped_func
def test_context_supports_fixtures():
with attest.raises(Thing.DoesNotExist):
Thing.objects.get(name="loaded from fixture")
manager = contextmanager(TestContext(fixtures=["tests"]))
with manager():
Thing.objects.get(name="loaded from fixture")
def view(f):
"""
Decorate a view function.
The decorated function has to recieve a Page object
as a first parameter.
The resulting function will recieve all parameters *except*
the Page object, and will always return the rendered Page.
A generator function can be decorated for a different creature.
This results in a custom element, registered on all Page objects,
That can be used using `with` as normal.
This usage returns None.
Example usage:
@view
def main(p, title):
with p.div(klass='container'):
with p.h1():
p.text(title)
>>> main('Foo')
>>> '<div class="container"><h1>Foo</h1></div>'
"""
if is_gen(f):
Page.register(ctx.contextmanager(f))
return
@wraps(f)
def _inner(*args, **kwargs):
p = Page()
f(p, *args, **kwargs)
return str(p)
return _inner
def test_contextmanager(self):
"""Test unwrapping a context manager."""
self.assertEqual(
unwrap_function(contextmanager(my_function)),
my_function
)
def context(self, func):
"""Decorate a function to act as a context.
:param function func: the function that describes the context
"""
func = contextmanager(func)
self._contexts.append(func)
def make_connect_cmd(con):
def go(opts=None):
mc = con(opts)
try:
yield mc
finally:
mc.disconnect_all()
return contextmanager(go)
def bn_impossible():
criterion = bn_impossible_criterion()
image = bn_image()
label = bn_label()
cm_model = contextmanager(bn_model)
with cm_model() as model:
yield Adversarial(model, criterion, image, label)
def gl_bn_adversarial():
criterion = bn_criterion()
image = bn_image()
label = bn_label()
cm_model = contextmanager(gl_bn_model)
with cm_model() as model:
yield Adversarial(model, criterion, image, label)
def test_context_does_transaction_rollback():
manager = contextmanager(TestContext())
with manager():
# create a thing, but it should be rolled back when the context exits
thing = Thing.objects.create(name="foo")
with attest.raises(Thing.DoesNotExist):
Thing.objects.get(pk=thing.pk)
def gl_bn_model():
"""Same as bn_model but without gradient.
"""
cm_model = contextmanager(bn_model)
with cm_model() as model:
model = GradientLess(model)
yield model
def main():
parser = argparse.ArgumentParser()
parser = argparse.ArgumentParser(description="Bibliography database manipulation")
parser.add_argument("--debug", action="store_true")
parser.add_argument("--logging-level", "-L",
help="Logging level: CRITICAL, ERROR, WARNING, INFO, DEBUG",
metavar="LEVEL", type=str, default="WARNING")
parser.add_argument("--data-dir", help="Path to articles directory", type=str, default=None)
subparsers = parser.add_subparsers(title='Commands', dest='_commandName')
for cmdType in Registry.commands:
cmdType().args(subparsers)
argcomplete.autocomplete(parser)
args = parser.parse_args()
if args.debug:
msg.setup(level="DEBUG")
try:
from ipdb import launch_ipdb_on_exception
except ModuleNotFoundError:
from contextlib import contextmanager
def noop():
yield
launch_ipdb_on_exception = contextmanager(noop)
else:
msg.setup(level=args.logging_level)
ddir = Database.getDataDir(dataDir=args.data_dir)
if ddir:
RequestCache(os.path.join(ddir, ".cache.pkl"))
try:
if hasattr(args, "func"):
if args.debug:
with launch_ipdb_on_exception():
args.func(args)
else:
args.func(args)
else:
parser.print_usage()
except UserException as e:
msg.error("Error: %s", e)
sys.exit(1)
except AbortException:
msg.error("Aborted")
sys.exit(1)
except (WorkExistsException, RepositoryException) as e:
msg.error(str(e))
sys.exit(1)
except:
t,v,_ = sys.exc_info()
msg.critical("Unhandled exception: %s(%s)", t.__name__, v)
sys.exit(1)
def __call__(self, filename, *args, **kwargs):
print('MockOpen called')
if filename.startswith(self.test_dir):
if filename not in self.files:
print('Mocking %s' % filename)
self.files[filename] = StringIO.StringIO()
return contextlib.contextmanager(yields)(self.files[filename])
else:
return self.old_open(filename, *args, **kwargs)
def reentrentcontext(func):
context = contextmanager(func)
entered = []
def decorated(*a, **k):
if entered == []:
entered.append(True) # I miss nonlocal
return context(*a, **k)
return noop_context()
return decorated
def bn_adversarial_mae():
criterion = bn_criterion()
image = bn_image()
label = bn_label()
distance = MAE
cm_model = contextmanager(bn_model)
with cm_model() as model:
yield Adversarial(model, criterion, image, label, distance=distance)
def setUp(self):
self.a = A()
self.a.x = 0
def incr():
self.a.x += 1
yield
self.a.x += 1
self.incr = contextmanager(incr)
self.with_lk = with_(self.incr, lambda: None)
def NewCookieJar(self):
"""Makes a context manager that sets up a cookie jar for DoGet/DoPost."""
def SetCookieJar():
original_cookie_jar = self.cookie_jar
self.cookie_jar = cookielib.CookieJar()
try:
yield self.cookie_jar
finally:
self.cookie_jar = original_cookie_jar
return contextlib.contextmanager(SetCookieJar)()
def _wrapper(func):
func.renders_operations = operations
if any(isinstance(op, ChildlessOperation) for op in operations):
if not all(isinstance(op, ChildlessOperation) for op in operations):
raise Exception("Cannot mix ChildlessOperations and normal Operations")
return func
return contextlib.contextmanager(func)
def subTest(test, *args, **kwargs):
"""Compatibility shim for unittest.TestCase.subTest.
Usage: ``with tornado.test.util.subTest(self, x=x):``
"""
try:
subTest = test.subTest # py34+
except AttributeError:
subTest = contextlib.contextmanager(lambda *a, **kw: (yield))
return subTest(*args, **kwargs)
def decorator(predicate):
context = contextmanager(predicate)
@wraps(predicate)
def take_arguments(*pargs, **pkwargs):
@wraps(predicate)
def decorator(cls):
if inspect.isclass(cls):
orig_setup = cls.setUp
orig_teardown = cls.tearDown
@wraps(cls.setUp)
def around_setup(*args, **kwargs):
try:
contexts = args[0].__rb3dc_contexts__
except AttributeError:
contexts = args[0].__rb3dc_contexts__ = []
p = context(*pargs, **pkwargs)
p.__enter__()
contexts.append(p)
return orig_setup(*args, **kwargs)
around_setup.__wrapped__ = cls.setUp
cls.setUp = around_setup
@wraps(cls.tearDown)
def around_teardown(*args, **kwargs):
try:
contexts = args[0].__rb3dc_contexts__
except AttributeError:
pass
else:
for context in contexts:
context.__exit__(*sys.exc_info())
orig_teardown(*args, **kwargs)
around_teardown.__wrapped__ = cls.tearDown
cls.tearDown = around_teardown
return cls
else:
@wraps(cls)
def around_case(self, *args, **kwargs):
with context(*pargs, **pkwargs) as context_args:
context_args = context_args or ()
if not isinstance(context_args, tuple):
context_args = (context_args,)
return cls(*(self,) + args + context_args, **kwargs)
return around_case
if len(pargs) == 1 and callable(pargs[0]):
fun, pargs = pargs[0], ()
return decorator(fun)
return _CallableContext(context, pargs, pkwargs, decorator)
assert take_arguments.__wrapped__
return take_arguments
def template_rendering_tracking_works():
manager = contextmanager(TestContext())
with manager() as client:
response = client.get("/")
assert response.content == b"rendered from template.html\n"
if hasattr(response, "templates"):
# Django >= 1.3
assert [t.name for t in response.templates] == ["template.html"]
else:
# Django <= 1.2
assert response.template.name == "template.html"
def make_contextmanager(fn):
if inspect.isgeneratorfunction(fn):
return contextmanager(fn)
if fn is None:
fn = lambda *a, **kw: None
@contextmanager
@functools.wraps(fn)
def wrapper(*a, **kw):
yield fn(*a, **kw)
return wrapper
def __call__(self, filename, mode, *args, **kwargs):
if filename.startswith(self.test_dir):
if filename not in self.files or mode in ('w', 'w+'):
self.files[filename] = StringIO.StringIO()
fakefile = self.files[filename]
if mode in ('r', 'r+'):
fakefile.seek(0)
else:
fakefile.seek(0, os.SEEK_END)
return contextlib.contextmanager(yields)(fakefile)
else:
return self.old_open(filename, *args, **kwargs)
def patch_modules_registry():
"""Backport the Registry.cursor helper."""
def cursor(self, auto_commit=True):
cr = self.db.cursor()
try:
yield cr
if auto_commit:
cr.commit()
finally:
cr.close()
openerp.modules.registry.Registry.cursor = contextmanager(cursor)
def test_collectstats():
"""
Some extra tests for `collectstats`.
"""
# Test for getting extra data even if call raised an exception.
class RaiseException(object):
@staticmethod
def __extra_stats__():
return {"hello", "world!"}
@collectstats(collect_extra=True)
def doit(self):
raise NotImplementedError("And never will be!")
recorder = ApiCallsRecorder(suspend_save=True)
with recorder:
try:
RaiseException().doit()
assert True, "Call to RaiseException().doit() should have raised an exception"
except NotImplementedError:
pass
assert recorder.api_stats[0].extra_stats == {"hello", "world!"}
# Test for objects with `__getattr__`.
class IHazGetAttr(object):
def __getattr__(self, item):
return self
@collectstats(collect_extra=True)
def bar(self):
return self.__name__
recorder = ApiCallsRecorder(suspend_save=True)
with recorder:
IHazGetAttr().bar()
assert recorder.api_stats[0].extra_stats == "TypeError(\"'IHazGetAttr' object is not callable\")"
# Test that there are no new lines in the json dumps of saved ApiCalls. Newlines will mess up reading the file.
class LotsOfNewLines(object):
def __extra_stats__(self):
return "\nHello,\nWorld\n!\n"
@collectstats("\nClass\nName\n", collect_extra=True)
def bar(self, a, b):
return "\nYes\nHello\n" + a + b
import StringIO
import mock
stringio = StringIO.StringIO() # ContextStringIO()
with mock.patch.object(ApiCallsRecorder, "_get_save_file",
new=staticmethod(contextmanager(lambda: (yield stringio)))):
LotsOfNewLines().bar("\nA\n", b="\nB\n")
# TODO: separate writing logic to a different class so that we don't test implementation details here.
assert all('\n' not in buf for buf in stringio.buflist[:-1])
assert '\n' not in stringio.buflist[-1][:-1]
assert stringio.buflist[-1][-1] == '\n'
def __init__(self):
self.count = 0
self.func_cm = contextmanager(func)
self._lock = threading.RLock()
test_cluster.ddl_check_query(
instance,
"CREATE TABLE test_atomic.rvcmt ON CLUSTER cluster (n UInt64, m Int8, k UInt64) ENGINE=ReplicatedVersionedCollapsingMergeTree(m, k) ORDER BY n"
)
test_cluster.ddl_check_query(
instance, "DROP DATABASE test_atomic ON CLUSTER cluster")
test_cluster.ddl_check_query(
instance,
"CREATE DATABASE test_ordinary ON CLUSTER cluster ENGINE=Ordinary")
assert "are supported only for ON CLUSTER queries with Atomic database engine" in \
instance.query_and_get_error("CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n")
assert "are supported only for ON CLUSTER queries with Atomic database engine" in \
instance.query_and_get_error("CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{uuid}/', '{replica}') ORDER BY n")
test_cluster.ddl_check_query(
instance,
"CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{table}/', '{replica}') ORDER BY n"
)
assert instance.query("SHOW CREATE test_ordinary.rmt FORMAT TSVRaw") == \
"CREATE TABLE test_ordinary.rmt\n(\n `n` UInt64,\n `s` String\n)\nENGINE = ReplicatedMergeTree('/{shard}/rmt/', '{replica}')\nORDER BY n\nSETTINGS index_granularity = 8192\n"
test_cluster.ddl_check_query(
instance, "DROP DATABASE test_ordinary ON CLUSTER cluster")
test_cluster.pm_random_drops.push_rules(rules)
if __name__ == '__main__':
with contextmanager(test_cluster)() as ctx_cluster:
for name, instance in list(ctx_cluster.instances.items()):
print(name, instance.ip_address)
input("Cluster created, press any key to destroy...")
def binarized2_bn_model_fixture():
cm_model = contextmanager(binarized2_bn_model)
with cm_model() as model:
yield model
def main():
parser = argparse.ArgumentParser()
parser = argparse.ArgumentParser(
description="Bibliography database manipulation")
parser.add_argument("--debug", action="store_true")
parser.add_argument(
"--logging-level",
"-L",
help="Logging level: CRITICAL, ERROR, WARNING, INFO, DEBUG",
metavar="LEVEL",
type=str,
default="WARNING")
parser.add_argument("--data-dir",
help="Path to articles directory",
type=str,
default=None)
subparsers = parser.add_subparsers(title='Commands', dest='_commandName')
for cmdType in Registry.commands:
cmdType().args(subparsers)
argcomplete.autocomplete(parser)
args = parser.parse_args()
if args.debug:
msg.setup(level="DEBUG")
try:
from ipdb import launch_ipdb_on_exception
except ModuleNotFoundError:
from contextlib import contextmanager
def noop():
yield
launch_ipdb_on_exception = contextmanager(noop)
else:
msg.setup(level=args.logging_level)
ddir = Database.getDataDir(dataDir=args.data_dir)
if ddir:
RequestCache(os.path.join(ddir, ".cache.pkl"))
try:
if hasattr(args, "func"):
if args.debug:
with launch_ipdb_on_exception():
args.func(args)
else:
args.func(args)
else:
parser.print_usage()
except UserException as e:
msg.error("Error: %s", e)
sys.exit(1)
except AbortException:
msg.error("Aborted")
sys.exit(1)
except (WorkExistsException, RepositoryException) as e:
msg.error(str(e))
sys.exit(1)
except:
t, v, _ = sys.exc_info()
msg.critical("Unhandled exception: %s(%s)", t.__name__, v)
sys.exit(1)
def create_slack_finding_sender(args, db_file):
if not args.cache_only and args.slack_webhook:
return SlackFindingSender(args.slack_webhook, db_file)
else:
return contextmanager(lambda: iter([None]))()
import contextlib
import logging
import numpy as np
import nengo.utils.numpy as npext
from nengo.builder import Builder
from nengo.connection import Connection
from nengo.ensemble import Ensemble
from nengo.network import Network
from nengo.node import Node
from nengo.probe import Probe
from nengo.utils.progress import Progress
logger = logging.getLogger(__name__)
nullcontext = contextlib.contextmanager(lambda: (yield))
@Builder.register(Network) # noqa: C901
def build_network(model, network, progress=None):
"""Builds a `.Network` object into a model.
The network builder does this by mapping each high-level object to its
associated signals and operators one-by-one, in the following order:
1. Ensembles, nodes, neurons
2. Subnetworks (recursively)
3. Connections, learning rules
4. Probes
Before calling any of the individual objects' build functions, random
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory, check_mongo_calls
from openedx.core.djangolib.testing.utils import skip_unless_lms
from common.djangoapps.student.tests.factories import AdminFactory, UserFactory
from .. import DEFAULT_FIELDS, OPTIONAL_FIELDS, PathItem
from ..models import Bookmark, XBlockCache, parse_path_data
from .factories import BookmarkFactory
EXAMPLE_USAGE_KEY_1 = 'i4x://org.15/course_15/chapter/Week_1'
EXAMPLE_USAGE_KEY_2 = 'i4x://org.15/course_15/chapter/Week_2'
noop_contextmanager = contextmanager(lambda x: (yield)) # pylint: disable=invalid-name
class BookmarksTestsBase(ModuleStoreTestCase):
"""
Test the Bookmark model.
"""
ALL_FIELDS = DEFAULT_FIELDS + OPTIONAL_FIELDS
STORE_TYPE = ModuleStoreEnum.Type.split
TEST_PASSWORD = '******'
ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache']
def setUp(self):
super().setUp()
def documented_contextmanager(func):
wrapper = contextmanager(func)
wrapper.undecorated = func
return wrapper
def __init__(self, func):
self.count = 0
self.func_cm = contextmanager(func)
self._lock = RLock()
def eg_bn_model():
cm_model = contextmanager(bn_model)
with cm_model() as model:
gradient_estimator = GradientEstimator(epsilon=0.01)
model = ModelWithEstimatedGradients(model, gradient_estimator)
yield model
def run(dataset: Dataset, config: TaskConfig):
log.info("\n**** H2O AutoML ****\n")
# Mapping of benchmark metrics to H2O metrics
metrics_mapping = dict(acc='mean_per_class_error',
auc='AUC',
logloss='logloss',
mae='mae',
mse='mse',
r2='r2',
rmse='rmse',
rmsle='rmsle')
sort_metric = metrics_mapping[
config.metric] if config.metric in metrics_mapping else None
if sort_metric is None:
# TODO: Figure out if we are going to blindly pass metrics through, or if we use a strict mapping
log.warning("Performance metric %s not supported, defaulting to AUTO.",
config.metric)
try:
training_params = {
k: v
for k, v in config.framework_params.items()
if not k.startswith('_')
}
nthreads = config.framework_params.get('_nthreads', config.cores)
jvm_memory = str(
round(config.max_mem_size_mb * 2 /
3)) + "M" # leaving 1/3rd of available memory for XGBoost
log.info("Starting H2O cluster with %s cores, %s memory.", nthreads,
jvm_memory)
max_port_range = 49151
min_port_range = 1024
rnd_port = os.getpid() % (max_port_range -
min_port_range) + min_port_range
port = config.framework_params.get('_port', rnd_port)
h2o.init(
nthreads=nthreads,
port=port,
min_mem_size=jvm_memory,
max_mem_size=jvm_memory,
strict_version_check=config.framework_params.get(
'_strict_version_check', True)
# log_dir=os.path.join(config.output_dir, 'logs', config.name, str(config.fold))
)
# Load train as an H2O Frame, but test as a Pandas DataFrame
log.debug("Loading train data from %s.", dataset.train.path)
train = h2o.import_file(dataset.train.path,
destination_frame=frame_name('train', config))
# train.impute(method='mean')
log.debug("Loading test data from %s.", dataset.test.path)
test = h2o.import_file(dataset.test.path,
destination_frame=frame_name('test', config))
# test.impute(method='mean')
log.info("Running model on task %s, fold %s.", config.name,
config.fold)
log.debug(
"Running H2O AutoML with a maximum time of %ss on %s core(s), optimizing %s.",
config.max_runtime_seconds, config.cores, sort_metric)
aml = H2OAutoML(
max_runtime_secs=config.max_runtime_seconds,
max_runtime_secs_per_model=round(
config.max_runtime_seconds /
2), # to prevent timeout on ensembles
sort_metric=sort_metric,
seed=config.seed,
**training_params)
monitor = (
BackendMemoryMonitoring(
frequency_seconds=rconfig().monitoring.frequency_seconds,
check_on_exit=True,
verbosity=rconfig().monitoring.verbosity)
if config.framework_params.get('_monitor_backend', False)
# else contextlib.nullcontext # Py 3.7+ only
else contextlib.contextmanager(iter)([0]))
with Timer() as training:
with monitor:
aml.train(y=dataset.target.index, training_frame=train)
if not aml.leader:
raise NoResultError(
"H2O could not produce any model in the requested time.")
save_predictions(aml, test, dataset=dataset, config=config)
save_artifacts(aml, dataset=dataset, config=config)
return dict(models_count=len(aml.leaderboard),
training_duration=training.duration)
finally:
if h2o.connection():
# h2o.remove_all()
h2o.connection().close()
if h2o.connection().local_server:
h2o.connection().local_server.shutdown()
def get_data_from_url(url: str):
return request.urlopen(url)
def _change_directory(destination_directory):
cwd = os.getcwd()
os.chdir(destination_directory)
try:
yield
except:
pass
finally:
os.chdir(cwd)
cd = contextlib.contextmanager(_change_directory)
def get_indra_statements_from_directory(directory: str) -> Iterable[Influence]:
""" Returns a list of INDRA statements from a directory containing JSON-LD
output from Eidos. """
return chain.from_iterable(
map(
lambda ep: ep.statements,
map(eidos.process_json_ld_file, tqdm(glob(directory))),
))
def grouper(iterable, n, fillvalue=None):
"Collect data into fixed-length chunks or blocks"
# grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
class WMConnectionBase(object):
"""
Generic db connection and transaction methods used by all of the WMCore classes.
"""
def __init__(self, daoPackage, logger=None, dbi=None):
"""
___init___
Initialize all the database connection attributes and the logging
attritbutes. Create a DAO factory for given daoPackage as well. Finally,
check to see if a transaction object has been created. If none exists,
create one but leave the transaction closed.
"""
myThread = threading.currentThread()
if logger:
self.logger = logger
else:
self.logger = myThread.logger
if dbi:
self.dbi = dbi
else:
self.dbi = myThread.dbi
self.daofactory = DAOFactory(package=daoPackage,
logger=self.logger,
dbinterface=self.dbi)
if "transaction" not in dir(myThread):
myThread.transaction = Transaction(self.dbi)
return
def getDBConn(self):
"""
_getDBConn_
Retrieve the database connection that is associated with the current
dataabase transaction.
It transaction exists, it will return connection
which that transaction belong to.
This won't create the transaction if it doesn't exist, it will just return
None.
"""
myThread = threading.currentThread()
if "transaction" not in dir(myThread):
return None
return myThread.transaction.conn
def beginTransaction(self):
"""
_beginTransaction_
Begin a database transaction if one does not already exist.
"""
myThread = threading.currentThread()
if "transaction" not in dir(myThread):
myThread.transaction = Transaction(self.dbi)
return False
if myThread.transaction.transaction == None:
myThread.transaction.begin()
return False
return True
def existingTransaction(self):
"""
_existingTransaction_
Return True if there is an open transaction, False otherwise.
"""
myThread = threading.currentThread()
if "transaction" not in dir(myThread):
return False
elif myThread.transaction.transaction != None:
return True
return False
def commitTransaction(self, existingTransaction):
"""
_commitTransaction_
Commit a database transaction that was begun by self.beginTransaction().
"""
if not existingTransaction:
myThread = threading.currentThread()
myThread.transaction.commit()
return
def __getstate__(self):
"""
__getstate__
The database connection information isn't pickleable, so we to kill that
before we attempt to pickle.
"""
self.dbi = None
self.logger = None
self.daofactory = None
return self.__dict__
def transactionContext(self):
"""
Returns a transaction as a ContextManager
Usage:
with transactionContext():
databaseCode1()
databaseCode2()
Equates to beginTransaction() followed by either
commitTransaction or a rollback
"""
existingTransaction = self.beginTransaction()
try:
yield existingTransaction
except:
# responsibility for rolling back is on the transaction starter
if not existingTransaction:
self.logger.error('Exception caught, rolling back transaction')
threading.currentThread().transaction.rollback()
raise
else:
# only commits if transaction started by this invocation
self.commitTransaction(existingTransaction)
try:
transactionContext = contextmanager(transactionContext)
except NameError:
pass
def rollbackTransaction(self, existingTransaction):
"""Rollback transaction if we started it"""
if not existingTransaction:
threading.currentThread().transaction.rollback()
__BUCKET_ERRORS = []
def threshold_suppress():
"""Suppress errors as long as they stay below a threshold."""
try:
yield
except exceptions, ex:
ts = int(time.time())
timebucket = ts - (ts % interval)
if __BUCKET_ERRORS:
(lastbucket, errors) = __BUCKET_ERRORS[0]
else:
lastbucket = None
if lastbucket != timebucket:
errors = defaultdict(int)
__BUCKET_ERRORS.insert(0, (timebucket, errors))
while len(__BUCKET_ERRORS) > 1:
__BUCKET_ERRORS.pop()
errors[type(ex)] += 1
if errors[type(ex)] < threshold:
stats.incr('%s_suppressed' % type(ex).__name__)
logging.exception("Suppressing error: %s", ex)
return
logging.debug("Too many %s errors, raising", type(ex))
stats.incr('%s_suppress_failures' % type(ex).__name__)
raise
return contextmanager(threshold_suppress)
def get_install_actions(prefix,
specs,
env,
retries=0,
subdir=None,
verbose=True,
debug=False,
locking=True,
bldpkgs_dirs=None,
timeout=90,
disable_pip=False,
max_env_retry=3,
output_folder=None,
channel_urls=None):
global cached_actions
global last_index_ts
actions = {}
log = utils.get_logger(__name__)
conda_log_level = logging.WARN
specs = list(specs)
if verbose:
capture = contextlib.contextmanager(lambda: (yield))
elif debug:
capture = contextlib.contextmanager(lambda: (yield))
conda_log_level = logging.DEBUG
else:
capture = utils.capture
for feature, value in feature_list:
if value:
specs.append('%s@' % feature)
bldpkgs_dirs = ensure_list(bldpkgs_dirs)
index, index_ts = get_build_index(subdir,
list(bldpkgs_dirs)[0],
output_folder=output_folder,
channel_urls=channel_urls,
debug=debug,
verbose=verbose,
locking=locking,
timeout=timeout)
specs = tuple(utils.ensure_valid_spec(spec) for spec in specs)
if ((specs, env, subdir, channel_urls, disable_pip) in cached_actions
and last_index_ts >= index_ts):
actions = cached_actions[(specs, env, subdir, channel_urls,
disable_pip)].copy()
if "PREFIX" in actions:
actions['PREFIX'] = prefix
elif specs:
# this is hiding output like:
# Fetching package metadata ...........
# Solving package specifications: ..........
with utils.LoggingContext(conda_log_level):
with capture():
try:
actions = install_actions(prefix, index, specs, force=True)
except NoPackagesFoundError as exc:
raise DependencyNeedsBuildingError(exc, subdir=subdir)
except (SystemExit, PaddingError, LinkError,
DependencyNeedsBuildingError, CondaError,
AssertionError) as exc:
if 'lock' in str(exc):
log.warn(
"failed to get install actions, retrying. exception was: %s",
str(exc))
elif ('requires a minimum conda version' in str(exc)
or 'link a source that does not' in str(exc)
or isinstance(exc, AssertionError)):
locks = utils.get_conda_operation_locks(
locking, bldpkgs_dirs, timeout)
with utils.try_acquire_locks(locks, timeout=timeout):
pkg_dir = str(exc)
folder = 0
while os.path.dirname(
pkg_dir) not in pkgs_dirs and folder < 20:
pkg_dir = os.path.dirname(pkg_dir)
folder += 1
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if pkg_dir in pkgs_dirs and os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retries < max_env_retry:
log.warn(
"failed to get install actions, retrying. exception was: %s",
str(exc))
actions = get_install_actions(
prefix,
tuple(specs),
env,
retries=retries + 1,
subdir=subdir,
verbose=verbose,
debug=debug,
locking=locking,
bldpkgs_dirs=tuple(bldpkgs_dirs),
timeout=timeout,
disable_pip=disable_pip,
max_env_retry=max_env_retry,
output_folder=output_folder,
channel_urls=tuple(channel_urls))
else:
log.error(
"Failed to get install actions, max retries exceeded."
)
raise
if disable_pip:
for pkg in ('pip', 'setuptools', 'wheel'):
# specs are the raw specifications, not the conda-derived actual specs
# We're testing that pip etc. are manually specified
if not any(
re.match('^%s(?:$| .*)' % pkg, str(dep))
for dep in specs):
actions['LINK'] = [
spec for spec in actions['LINK'] if spec.name != pkg
]
utils.trim_empty_keys(actions)
cached_actions[(specs, env, subdir, channel_urls,
disable_pip)] = actions.copy()
last_index_ts = index_ts
return actions
def gl_bn_model_fixture():
cm_model = contextmanager(gl_bn_model)
with cm_model() as model:
yield model
async def fixture(unused_port_factory):
make_server = contextlib.contextmanager(_server_fixture(server))
with make_server(unused_port_factory) as port:
async with channel(f"localhost:{port}") as chan:
yield stub(chan)
instance = cluster.instances['ch3']
ddl_check_query(
instance,
"DROP TABLE IF EXISTS test_optimize ON CLUSTER cluster FORMAT TSV")
ddl_check_query(
instance,
"CREATE TABLE test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)"
)
ddl_check_query(
instance, "OPTIMIZE TABLE test_optimize ON CLUSTER cluster FORMAT TSV")
def test_create_as_select(started_cluster):
instance = cluster.instances['ch2']
ddl_check_query(
instance,
"CREATE TABLE test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)"
)
assert TSV(instance.query(
"SELECT x FROM test_as_select ORDER BY x")) == TSV("1\n2\n")
ddl_check_query(instance,
"DROP TABLE IF EXISTS test_as_select ON CLUSTER cluster")
if __name__ == '__main__':
with contextmanager(started_cluster)() as cluster:
for name, instance in cluster.instances.items():
print name, instance.ip_address
raw_input("Cluster created, press any key to destroy...")
'hazards': [5, 9]
}
}
object = sampledb.logic.objects.create_object(
action_id=instrument_action.id,
data=data,
user_id=user.id,
previous_object_id=None,
schema=schema)
os.makedirs('docs/static/img/generated', exist_ok=True)
options = Options()
# disable Chrome sandbox for root in GitLab CI
if 'CI' in os.environ and getpass.getuser() == 'root':
options.add_argument('--no-sandbox')
with contextlib.contextmanager(
tests.conftest.create_flask_server)(app) as flask_server:
with contextlib.closing(Chrome(options=options)) as driver:
time.sleep(5)
object_permissions(flask_server.base_url, driver)
default_permissions(flask_server.base_url, driver)
guest_invitation(flask_server.base_url, driver)
action(flask_server.base_url, driver, instrument_action)
hazards_input(flask_server.base_url, driver, instrument_action)
tags_input(flask_server.base_url, driver, object)
comments(flask_server.base_url, driver, object)
activity_log(flask_server.base_url, driver, object)
files(flask_server.base_url, driver, object)
file_information(flask_server.base_url, driver, object)
labels(flask_server.base_url, driver, object)
advanced_search_by_property(flask_server.base_url, driver,
object)
def get_build_index(config, subdir, clear_cache=False, omit_defaults=False):
global local_index_timestamp
global local_subdir
global cached_index
global cached_channels
log = utils.get_logger(__name__)
mtime = 0
if config.output_folder:
output_folder = config.output_folder
else:
output_folder = os.path.dirname(config.bldpkgs_dir)
# check file modification time - this is the age of our index.
index_file = os.path.join(output_folder, subdir, 'repodata.json')
if os.path.isfile(index_file):
mtime = os.path.getmtime(index_file)
if (clear_cache or not os.path.isfile(index_file) or local_subdir != subdir
or mtime > local_index_timestamp
or cached_channels != config.channel_urls):
log.debug(
"Building new index for subdir '{}' with channels {}, condarc channels "
"= {}".format(subdir, config.channel_urls, not omit_defaults))
# priority: local by croot (can vary), then channels passed as args,
# then channels from config.
capture = contextlib.contextmanager(lambda: (yield))
if config.debug:
log_context = partial(utils.LoggingContext, logging.DEBUG)
elif config.verbose:
log_context = partial(utils.LoggingContext, logging.INFO)
else:
log_context = partial(utils.LoggingContext, logging.CRITICAL + 1)
capture = utils.capture
urls = list(config.channel_urls)
if os.path.isdir(output_folder):
urls.insert(0, url_path(output_folder))
ensure_valid_channel(output_folder, subdir, config)
# silence output from conda about fetching index files
with log_context():
with capture():
# replace noarch with native subdir - this ends up building an index with both the
# native content and the noarch content.
if subdir == 'noarch':
subdir = conda_interface.subdir
try:
cached_index = get_index(channel_urls=urls,
prepend=not omit_defaults,
use_local=False,
use_cache=False,
platform=subdir)
# HACK: defaults does not have the many subfolders we support. Omit it and
# try again.
except CondaHTTPError:
if 'defaults' in urls:
urls.remove('defaults')
cached_index = get_index(channel_urls=urls,
prepend=omit_defaults,
use_local=False,
use_cache=False,
platform=subdir)
local_index_timestamp = mtime
local_subdir = subdir
cached_channels = config.channel_urls
return cached_index, local_index_timestamp
def wrapper(patcher):
patcher = contextmanager(patcher)
cls.INTENSIVE_CALLS_PATCHER[metric_name] = patcher
return patcher
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, declarative_base
import settings
engine = create_engine(
'postgresql://{}:{}@{}:{}/{}'.format(
settings.global_settings.db_user,
settings.global_settings.db_password,
settings.global_settings.db_host,
settings.global_settings.db_port,
settings.global_settings.db_name,
)
)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base = declarative_base()
def get_session():
session = SessionLocal()
try:
yield session
finally:
session.close()
context_session = contextmanager(get_session)
def eg_bn_model(request):
eg_bn_model = eg_bn_model_factory(request)
cm_model = contextmanager(eg_bn_model)
with cm_model() as model:
yield model
self.cur = cur
def __enter__(self):
# on enter create the generator
self.gen = temptable(self.cur)
next(self.gen)
def __exit__(self, *args):
# on exit go back into method, drop table
# and then return None
next(self.gen, None)
with connect('test.db') as conn:
cur = conn.cursor()
with contextmanager(cur):
cur.execute('insert into points (x, y) values (3, 3)')
cur.execute('insert into points (x, y) values (3, 2)')
cur.execute('insert into points (x, y) values (3, 1)')
cur.execute('insert into points (x, y) values (3, 4)')
for row in cur.execute('select x, y from points'):
print(row)
for row in cur.execute('select sum(x * y) from points'):
print(row)
# a more general solution is to pass the generator
# into the __init__
class contextmanager:
def __init__(self, gen):
self.gen = gen
def enter(self, fixtures, setup_callbacks=None, teardown_callbacks=None, stop_setups=False):
"""Transform each fixture_method into a context manager, enter them
recursively, and yield any failures.
`stop_setups` is set after a setup fixture fails. This flag prevents
more setup fixtures from being added to the onion after a failure as we
recurse through the list of fixtures.
"""
# base case
if not fixtures:
yield []
return
setup_callbacks = setup_callbacks or [None, None]
teardown_callbacks = teardown_callbacks or [None, None]
fixture = fixtures[0]
ctm = contextlib.contextmanager(fixture)()
# class_teardown fixture is wrapped as
# class_setup_teardown. We should not fire events for the
# setup phase of this fake context manager.
suppress_callbacks = bool(fixture._fixture_type in TEARDOWN_FIXTURES)
# if a previous setup fixture failed, stop running new setup
# fixtures. this doesn't apply to teardown fixtures, however,
# because behind the scenes they're setup_teardowns, and we need
# to run the (empty) setup portion in order to get the teardown
# portion later.
if not stop_setups or fixture._fixture_type in TEARDOWN_FIXTURES:
enter_failures = self.run_fixture(
fixture,
ctm.__enter__,
enter_callback=None if suppress_callbacks else setup_callbacks[0],
exit_callback=None if suppress_callbacks else setup_callbacks[1],
)
# keep skipping setups once we've had a failure
stop_setups = stop_setups or bool(enter_failures)
else:
# we skipped the setup, pretend like nothing happened.
enter_failures = []
with self.enter(fixtures[1:], setup_callbacks, teardown_callbacks, stop_setups) as all_failures:
all_failures += enter_failures or []
# need to only yield one failure
yield all_failures
# this setup fixture got skipped due to an earlier setup fixture
# failure, or failed itself. all of these fixtures are basically
# represented by setup_teardowns, but because we never ran this setup,
# we have nothing to do for teardown (if we did visit it here, that
# would have the effect of running the setup we just skipped), so
# instead bail out and move on to the next fixture on the stack.
if stop_setups and fixture._fixture_type in SETUP_FIXTURES:
return
# class_setup fixture is wrapped as
# class_setup_teardown. We should not fire events for the
# teardown phase of this fake context manager.
suppress_callbacks = bool(fixture._fixture_type in SETUP_FIXTURES)
# this is hack to finish the remainder of the context manager without
# calling contextlib's __exit__; doing that messes up the stack trace
# we end up with.
def exit():
try:
ctm.gen.next()
except StopIteration:
pass
exit_failures = self.run_fixture(
fixture,
exit,
enter_callback=None if suppress_callbacks else teardown_callbacks[0],
exit_callback=None if suppress_callbacks else teardown_callbacks[1],
)
all_failures += exit_failures or []
def daily_stats(self, data=sample_data):
def fake_fetch():
yield iter(data)
self._fetch_mock.side_effect = contextmanager(fake_fetch)
return readers.jhucsse_reader.daily_stats()
def run(dataset, config):
log.info(f"\n**** H2O AutoML [v{h2o.__version__}] ****\n")
save_metadata(config, version=h2o.__version__)
# Mapping of benchmark metrics to H2O metrics
metrics_mapping = dict(acc='mean_per_class_error',
auc='AUC',
logloss='logloss',
mae='mae',
mse='mse',
r2='r2',
rmse='rmse',
rmsle='rmsle')
sort_metric = metrics_mapping[
config.metric] if config.metric in metrics_mapping else None
if sort_metric is None:
# TODO: Figure out if we are going to blindly pass metrics through, or if we use a strict mapping
log.warning("Performance metric %s not supported, defaulting to AUTO.",
config.metric)
try:
training_params = {
k: v
for k, v in config.framework_params.items()
if not k.startswith('_')
}
nthreads = config.framework_params.get('_nthreads', config.cores)
jvm_memory = str(
round(config.max_mem_size_mb * 2 /
3)) + "M" # leaving 1/3rd of available memory for XGBoost
log.info("Starting H2O cluster with %s cores, %s memory.", nthreads,
jvm_memory)
max_port_range = 49151
min_port_range = 1024
rnd_port = os.getpid() % (max_port_range -
min_port_range) + min_port_range
port = config.framework_params.get('_port', rnd_port)
init_params = config.framework_params.get('_init', {})
if "logs" in config.framework_params.get('_save_artifacts', []):
init_params['ice_root'] = output_subdir("logs", config)
h2o.init(nthreads=nthreads,
port=port,
min_mem_size=jvm_memory,
max_mem_size=jvm_memory,
**init_params)
import_kwargs = {}
# Load train as an H2O Frame, but test as a Pandas DataFrame
log.debug("Loading train data from %s.", dataset.train.path)
train = None
if version.parse(h2o.__version__) >= version.parse(
"3.32.0.3"
): # previous versions may fail to parse correctly some rare arff files using single quotes as enum/string delimiters (pandas also fails on same datasets)
import_kwargs['quotechar'] = '"'
train = h2o.import_file(dataset.train.path,
destination_frame=frame_name(
'train', config),
**import_kwargs)
if not verify_loaded_frame(train, dataset):
h2o.remove(train)
train = None
import_kwargs['quotechar'] = "'"
if not train:
train = h2o.import_file(dataset.train.path,
destination_frame=frame_name(
'train', config),
**import_kwargs)
# train.impute(method='mean')
log.debug("Loading test data from %s.", dataset.test.path)
test = h2o.import_file(dataset.test.path,
destination_frame=frame_name('test', config),
**import_kwargs)
# test.impute(method='mean')
log.info("Running model on task %s, fold %s.", config.name,
config.fold)
log.debug(
"Running H2O AutoML with a maximum time of %ss on %s core(s), optimizing %s.",
config.max_runtime_seconds, config.cores, sort_metric)
aml = H2OAutoML(max_runtime_secs=config.max_runtime_seconds,
sort_metric=sort_metric,
seed=config.seed,
**training_params)
monitor = (
BackendMemoryMonitoring(
frequency_seconds=config.ext.monitoring.frequency_seconds,
check_on_exit=True,
verbosity=config.ext.monitoring.verbosity)
if config.framework_params.get('_monitor_backend', False)
# else contextlib.nullcontext # Py 3.7+ only
else contextlib.contextmanager(iter)([0]))
with utils.Timer() as training:
with monitor:
aml.train(y=dataset.target.index, training_frame=train)
if not aml.leader:
raise FrameworkError(
"H2O could not produce any model in the requested time.")
with utils.Timer() as predict:
preds = aml.predict(test)
preds = extract_preds(preds, test, dataset=dataset)
save_artifacts(aml, dataset=dataset, config=config)
return result(output_file=config.output_predictions_file,
predictions=preds.predictions,
truth=preds.truth,
probabilities=preds.probabilities,
probabilities_labels=preds.probabilities_labels,
models_count=len(aml.leaderboard),
training_duration=training.duration,
predict_duration=predict.duration)
finally:
if h2o.connection():
# h2o.remove_all()
h2o.connection().close()
if h2o.connection().local_server:
h2o.connection().local_server.shutdown()
def wrapper(*args, **kwargs):
with contextmanager(disabler)():
return func(*args, **kwargs)
class CorrectnessTest(keras_parameterized.TestCase):
@keras_parameterized.run_with_all_model_types
@keras_parameterized.run_all_keras_modes
def test_loss_correctness(self):
# Test that training loss is the same in eager and graph
# (by comparing it to a reference value in a deterministic case)
layers = [
keras.layers.Dense(3, activation='relu',
kernel_initializer='ones'),
keras.layers.Dense(2,
activation='softmax',
kernel_initializer='ones')
]
model = testing_utils.get_model_from_layers(layers, input_shape=(4, ))
model.compile(loss='sparse_categorical_crossentropy',
optimizer=rmsprop.RMSprop(learning_rate=0.001),
run_eagerly=testing_utils.should_run_eagerly(),
experimental_run_tf_function=testing_utils.
should_run_tf_function())
x = np.ones((100, 4))
np.random.seed(123)
y = np.random.randint(0, 1, size=(100, 1))
history = model.fit(x, y, epochs=1, batch_size=10)
self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
@keras_parameterized.run_with_all_model_types
@keras_parameterized.run_all_keras_modes
def test_loss_correctness_with_iterator(self):
# Test that training loss is the same in eager and graph
# (by comparing it to a reference value in a deterministic case)
layers = [
keras.layers.Dense(3, activation='relu',
kernel_initializer='ones'),
keras.layers.Dense(2,
activation='softmax',
kernel_initializer='ones')
]
model = testing_utils.get_model_from_layers(layers, input_shape=(4, ))
model.compile(loss='sparse_categorical_crossentropy',
optimizer=rmsprop.RMSprop(learning_rate=0.001),
run_eagerly=testing_utils.should_run_eagerly(),
experimental_run_tf_function=testing_utils.
should_run_tf_function())
x = np.ones((100, 4), dtype=np.float32)
np.random.seed(123)
y = np.random.randint(0, 1, size=(100, 1))
dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
dataset = dataset.repeat(100)
dataset = dataset.batch(10)
history = model.fit(dataset, epochs=1, steps_per_epoch=10)
self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
def test_loss_in_call(self):
class HasLoss(keras.layers.Layer):
def call(self, x):
self.add_loss(x)
return x
layer = HasLoss()
layer(1.) # Plain-value inputs are only valid in eager mode.
self.assertEqual(1, len(layer.losses))
@parameterized.named_parameters([
('_None', contextlib.contextmanager(lambda: iter([None])), 0., 4.),
('_0', lambda: keras.backend.learning_phase_scope(0), 4., 4.),
('_1', lambda: keras.backend.learning_phase_scope(1), 0., 0.),
])
def test_nested_model_learning_phase(self, nested_scope_fn,
expected_training_loss,
expected_validation_loss):
"""Tests that learning phase is correctly set in an intermediate layer."""
def _make_unregularized_model():
inputs = keras.Input((4, ))
# Zero out activations when `training=True`.
x = keras.layers.Dropout(1. - 1. / (1 << 24))(inputs)
x = keras.layers.Dense(
10,
activation='relu',
trainable=False,
bias_initializer='zeros',
kernel_initializer='ones')(
x) # Just sum together all the activations.
outputs = keras.layers.Dense(3)(x)
return keras.Model(inputs, outputs)
def _regularize_model(unregularized_model):
inputs = keras.Input(unregularized_model.inputs[0].shape[1:])
with nested_scope_fn():
logits = unregularized_model(inputs)
outputs = keras.activations.softmax(logits)
model = keras.Model(inputs, outputs)
# Regularize the most recent activations of a post-dropout layer.
sample_activations = unregularized_model.get_layer(
index=-2).get_output_at(-1)
regularization_loss = keras.backend.mean(sample_activations)
model.add_loss(regularization_loss)
model.add_metric(regularization_loss,
aggregation='mean',
name='regularization_loss')
return model
# Make and compile models.
model = _regularize_model(_make_unregularized_model())
model.compile('sgd', 'sparse_categorical_crossentropy')
# Prepare fake data.
x = np.ones((20, 4)).astype(np.float32)
y = np.random.randint(0, 3, size=(20, )).astype(np.int64)
dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(2)
evaluation_results = dict(
zip(model.metrics_names, model.evaluate(dataset)))
# Rate of dropout depends on the learning phase.
self.assertEqual(evaluation_results['regularization_loss'],
expected_validation_loss)
history = model.fit(dataset, epochs=2, validation_data=dataset).history
self.assertAllEqual(history['regularization_loss'],
[expected_training_loss] * 2)
self.assertAllEqual(history['val_regularization_loss'],
[expected_validation_loss] * 2)
