def MockRequest(
env,
*,
path='/mockrequest/',
routing=True,
multilang=True,
context=frozendict(),
cookies=frozendict(),
country_code=None,
website=None,
sale_order_id=None,
website_sale_current_pl=None,
):
lang_code = context.get('lang', env.context.get('lang', 'en_US'))
env = env(context=dict(context, lang=lang_code))
request = Mock(
# request
httprequest=Mock(
host='localhost',
path=path,
app=odoo.http.root,
environ=dict(
EnvironBuilder(path=path,
base_url=HttpCase.base_url()).get_environ(),
REMOTE_ADDR=HOST,
),
cookies=cookies,
referrer='',
),
type='http',
future_response=odoo.http.FutureResponse(),
params={},
redirect=env['ir.http']._redirect,
session=DotDict(
odoo.http.DEFAULT_SESSION,
geoip={'country_code': country_code},
sale_order_id=sale_order_id,
website_sale_current_pl=website_sale_current_pl,
),
geoip={},
db=None,
env=env,
registry=env.registry,
cr=env.cr,
uid=env.uid,
context=env.context,
lang=env['res.lang']._lang_get(lang_code),
website=website,
)
if website:
request.website_routing = website.id
# The following code mocks match() to return a fake rule with a fake
# 'routing' attribute (routing=True) or to raise a NotFound
# exception (routing=False).
#
# router = odoo.http.root.get_db_router()
# rule, args = router.bind(...).match(path)
# # arg routing is True => rule.endpoint.routing == {...}
# # arg routing is False => NotFound exception
router = MagicMock()
match = router.return_value.bind.return_value.match
if routing:
match.return_value[0].routing = {
'type': 'http',
'website': True,
'multilang': multilang
}
else:
match.side_effect = NotFound
with contextlib.ExitStack() as s:
odoo.http._request_stack.push(request)
s.callback(odoo.http._request_stack.pop)
s.enter_context(patch('odoo.http.root.get_db_router', router))
yield request
def test__subscribed_values_updated__subscription_receives_events(self):
assert (len(self._tags) == 6
), "Test needs to be updated to add additional data types"
values = set()
polling_interval = timedelta(milliseconds=500)
date_value = datetime.now(timezone.utc)
bool_value = False
double_value = math.pi
int_value = -13
string_value = "hello there"
uint64_value = 2**31 + 3
with contextlib.ExitStack() as exit_stack:
selection = exit_stack.enter_context(
self.tag_manager.create_selection(self._tags))
subscription = exit_stack.enter_context(
selection.create_subscription(
update_interval=polling_interval))
writer = exit_stack.enter_context(
self.tag_manager.create_writer(buffer_size=2))
subscription.tag_changed += lambda tag, reader: values.add(
(tag, reader))
tags = {t.data_type.name: t for t in self._tags}
writer.write(tags["BOOLEAN"].path, tags["BOOLEAN"].data_type,
bool_value)
writer.write(tags["DATE_TIME"].path, tags["DATE_TIME"].data_type,
date_value)
writer.write(tags["DOUBLE"].path, tags["DOUBLE"].data_type,
double_value)
writer.write(tags["INT32"].path, tags["INT32"].data_type,
int_value)
writer.write(tags["STRING"].path, tags["STRING"].data_type,
string_value)
writer.write(tags["UINT64"].path, tags["UINT64"].data_type,
uint64_value)
writer.send_buffered_writes()
# Wait for the subscription events to come in.
for _ in range(10):
if len(self._tags) == len(values):
break
time.sleep(polling_interval.total_seconds())
else:
assert len(self._tags) == len(values)
# Exactly one of each data type.
assert len(self._tags) == len(
values) # len(set(t.tag.data_type for t in values))
for (tag, reader) in values:
assert reader is not None
if tag.data_type == tbase.DataType.DOUBLE:
assert double_value == reader.read().value
break
elif tag.data_type == tbase.DataType.INT32:
assert int_value == reader.read().value
break
elif tag.data_type == tbase.DataType.STRING:
assert string_value == reader.read().value
break
elif tag.data_type == tbase.DataType.BOOLEAN:
assert bool_value == reader.read().value
break
elif tag.data_type == tbase.DataType.UINT64:
assert uint64_value == reader.read().value
break
elif tag.data_type == tbase.DataType.DATE_TIME:
assert date_value == reader.read().value
break
else:
assert False, "Unknown data type {}".format(tag.data_type)
break
def R(self, line, cell=None, local_ns=None):
"""
Execute code in R, optionally returning results to the Python runtime.
In line mode, this will evaluate an expression and convert the returned
value to a Python object. The return value is determined by rpy2's
behaviour of returning the result of evaluating the final expression.
Multiple R expressions can be executed by joining them with
semicolons::
In [9]: %R X=c(1,4,5,7); sd(X); mean(X)
Out[9]: array([ 4.25])
In cell mode, this will run a block of R code. The resulting value
is printed if it would be printed when evaluating the same code
within a standard R REPL.
Nothing is returned to python by default in cell mode::
In [10]: %%R
....: Y = c(2,4,3,9)
....: summary(lm(Y~X))
Call:
lm(formula = Y ~ X)
Residuals:
1 2 3 4
0.88 -0.24 -2.28 1.64
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) 0.0800 2.3000 0.035 0.975
X 1.0400 0.4822 2.157 0.164
Residual standard error: 2.088 on 2 degrees of freedom
Multiple R-squared: 0.6993,Adjusted R-squared: 0.549
F-statistic: 4.651 on 1 and 2 DF, p-value: 0.1638
In the notebook, plots are published as the output of the cell::
%R plot(X, Y)
will create a scatter plot of X bs Y.
If cell is not None and line has some R code, it is prepended to
the R code in cell.
Objects can be passed back and forth between rpy2 and python via the
-i -o flags in line::
In [14]: Z = np.array([1,4,5,10])
In [15]: %R -i Z mean(Z)
Out[15]: array([ 5.])
In [16]: %R -o W W=Z*mean(Z)
Out[16]: array([ 5., 20., 25., 50.])
In [17]: W
Out[17]: array([ 5., 20., 25., 50.])
The return value is determined by these rules:
* If the cell is not None (i.e., has contents), the magic returns None.
* If the final line results in a NULL value when evaluated
by rpy2, then None is returned.
* No attempt is made to convert the final value to a structured array.
Use %Rget to push a structured array.
* If the -n flag is present, there is no return value.
* A trailing ';' will also result in no return value as the last
value in the line is an empty string.
"""
args = parse_argstring(self.R, line)
# arguments 'code' in line are prepended to
# the cell lines
if cell is None:
code = ''
return_output = True
line_mode = True
else:
code = cell
return_output = False
line_mode = False
code = ' '.join(args.code) + code
# if there is no local namespace then default to an empty dict
if local_ns is None:
local_ns = {}
if args.converter is None:
converter = self.converter
else:
try:
converter = local_ns[args.converter]
except KeyError:
try:
converter = self.shell.user_ns[args.converter]
except KeyError:
raise NameError("name '%s' is not defined" %
args.converter)
if not isinstance(converter, Converter):
raise TypeError(
"'%s' must be a %s object (but it is a %s)." %
(args.converter, Converter, type(localconverter)))
if args.input:
for input in ','.join(args.input).split(','):
try:
val = local_ns[input]
except KeyError:
try:
val = self.shell.user_ns[input]
except KeyError:
raise NameError("name '%s' is not defined" % input)
with localconverter(converter) as cv:
ro.r.assign(input, val)
tmpd = self.setup_graphics(args)
text_output = ''
try:
if line_mode:
for line in code.split(';'):
text_result, result, visible = self.eval(line)
text_output += text_result
if text_result:
# The last line printed something to the console so
# we won't return it.
return_output = False
else:
text_result, result, visible = self.eval(code)
text_output += text_result
if visible:
with contextlib.ExitStack() as stack:
if self.cache_display_data:
stack.enter_context(
rpy2.rinterface_lib.callbacks.obj_in_module(
rpy2.rinterface_lib.callbacks,
'consolewrite_print',
self.write_console_regular))
ro.r.show(result)
text_output += self.flush()
except RInterpreterError as e:
# TODO: Maybe we should make this red or something?
print(e.stdout)
if not e.stdout.endswith(e.err):
print(e.err)
if tmpd:
rmtree(tmpd)
return
finally:
if self.device in ['png', 'svg']:
ro.r('dev.off()')
if text_output:
# display_data.append(('RMagic.R', {'text/plain':text_output}))
displaypub.publish_display_data(data={'text/plain': text_output},
source='RMagic.R')
# publish the R images
if self.device in ['png', 'svg']:
display_data, md = self.publish_graphics(tmpd, args.isolate_svgs)
for tag, disp_d in display_data:
displaypub.publish_display_data(data=disp_d,
source=tag,
metadata=md)
# kill the temporary directory - currently created only for "svg"
# and "png" (else it's None)
rmtree(tmpd)
if args.output:
with localconverter(converter) as cv:
for output in ','.join(args.output).split(','):
output_ipy = ro.globalenv.find(output)
self.shell.push({output: output_ipy})
# this will keep a reference to the display_data
# which might be useful to other objects who happen to use
# this method
if self.cache_display_data:
self.display_cache = display_data
# We're in line mode and return_output is still True,
# so return the converted result
if return_output and not args.noreturn:
if result is not ri.NULL:
with localconverter(converter) as cv:
res = cv.rpy2py(result)
return res
def nested(*contexts):
with contextlib.ExitStack() as stack:
yield [stack.enter_context(c) for c in contexts]
# contextlib_exitstack_callbacks_error.py
import contextlib
def callback(*args, **kwds):
print('closing callback({}, {})'.format(args, kwds))
try:
with contextlib.ExitStack() as stack:
stack.callback(callback, 'arg1', 'arg2')
stack.callback(callback, arg3='val3')
raise RuntimeError('thrown error')
except RuntimeError as err:
print('ERROR: {}'.format(err))
def exec_command(
command_str: str,
*,
stdin: Optional[IO[Any]] = None,
input: Optional[bytes] = None,
timeout: Optional[float] = None,
gnu_time: Optional[str] = None
) -> Tuple[Dict[str, Any], subprocess.Popen]:
if input is not None:
assert stdin is None
stdin = subprocess.PIPE # type: ignore
if gnu_time is not None:
context: Any = tempfile.NamedTemporaryFile(delete=True)
else:
context = contextlib.ExitStack(
) # TODO: we should use contextlib.nullcontext() if possible
with context as fh:
command = shlex.split(command_str)
if gnu_time is not None:
command = [gnu_time, '-f', '%M', '-o', fh.name, '--'] + command
if os.name == 'nt':
# HACK: without this encoding and decoding, something randomly fails with multithreading; see https://github.com/kmyk/online-judge-tools/issues/468
command = command_str.encode().decode() # type: ignore
begin = time.perf_counter()
# We need kill processes called from the "time" command using process groups. Without this, orphans spawn. see https://github.com/kmyk/online-judge-tools/issues/640
preexec_fn = None
if gnu_time is not None and os.name == 'posix':
preexec_fn = os.setsid
try:
proc = subprocess.Popen(command,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=sys.stderr,
preexec_fn=preexec_fn)
except FileNotFoundError:
logger.error('No such file or directory: %s', command)
sys.exit(1)
except PermissionError:
logger.error('Permission denied: %s', command)
sys.exit(1)
answer: Optional[bytes] = None
try:
answer, _ = proc.communicate(input=input, timeout=timeout)
except subprocess.TimeoutExpired:
pass
finally:
if preexec_fn is not None:
try:
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
except ProcessLookupError:
pass
else:
proc.terminate()
end = time.perf_counter()
memory: Optional[float] = None
if gnu_time is not None:
with open(fh.name) as fh1:
reported = fh1.read()
logger.debug('GNU time says:\n%s', reported)
if reported.strip() and reported.splitlines()[-1].isdigit():
memory = int(reported.splitlines()[-1]) / 1000
info = {
'answer': answer, # Optional[byte]
'elapsed': end - begin, # float, in second
'memory': memory, # Optional[float], in megabyte
}
return info, proc
def main():
parser = DownloaderArgumentParser()
parser.add_argument('--name', metavar='PAT[,PAT...]',
help='download only models whose names match at least one of the specified patterns')
parser.add_argument('--list', type=Path, metavar='FILE.LST',
help='download only models whose names match at least one of the patterns in the specified file')
parser.add_argument('--all', action='store_true', help='download all available models')
parser.add_argument('--print_all', action='store_true', help='print all available models')
parser.add_argument('--precisions', metavar='PREC[,PREC...]',
help='download only models with the specified precisions (actual for DLDT networks)')
parser.add_argument('-o', '--output_dir', type=Path, metavar='DIR',
default=Path.cwd(), help='path where to save models')
parser.add_argument('--cache_dir', type=Path, metavar='DIR',
help='directory to use as a cache for downloaded files')
parser.add_argument('--num_attempts', type=positive_int_arg, metavar='N', default=1,
help='attempt each download up to N times')
parser.add_argument('--progress_format', choices=('text', 'json'), default='text',
help='which format to use for progress reporting')
# unlike Model Converter, -jauto is not supported here, because CPU count has no
# relation to the optimal number of concurrent downloads
parser.add_argument('-j', '--jobs', type=positive_int_arg, metavar='N', default=1,
help='how many downloads to perform concurrently')
args = parser.parse_args()
def make_reporter(context):
return common.Reporter(context,
enable_human_output=args.progress_format == 'text',
enable_json_output=args.progress_format == 'json')
reporter = make_reporter(common.DirectOutputContext())
cache = NullCache() if args.cache_dir is None else DirCache(args.cache_dir)
models = common.load_models_from_args(parser, args)
failed_models = set()
if args.precisions is None:
requested_precisions = common.KNOWN_PRECISIONS
else:
requested_precisions = set(args.precisions.split(','))
unknown_precisions = requested_precisions - common.KNOWN_PRECISIONS
if unknown_precisions:
sys.exit('Unknown precisions specified: {}.'.format(', '.join(sorted(unknown_precisions))))
with contextlib.ExitStack() as exit_stack:
session_factory = ThreadSessionFactory(exit_stack)
if args.jobs == 1:
results = [download_model(reporter, args, cache, session_factory, requested_precisions, model)
for model in models]
else:
results = common.run_in_parallel(args.jobs,
lambda context, model: download_model(
make_reporter(context), args, cache, session_factory, requested_precisions, model),
models)
failed_models = {model.name for model, successful in zip(models, results) if not successful}
if failed_models:
reporter.print('FAILED:')
for failed_model_name in failed_models:
reporter.print(failed_model_name)
sys.exit(1)
def differentiable_loss(self, model, guide, *args, **kwargs):
# get batched, enumerated, to_funsor-ed traces from the guide and model
with plate(
size=self.num_particles
) if self.num_particles > 1 else contextlib.ExitStack(), enum(
first_available_dim=(-self.max_plate_nesting -
1) if self.max_plate_nesting else None):
guide_tr = trace(guide).get_trace(*args, **kwargs)
model_tr = trace(replay(model, trace=guide_tr)).get_trace(
*args, **kwargs)
# extract from traces all metadata that we will need to compute the elbo
guide_terms = terms_from_trace(guide_tr)
model_terms = terms_from_trace(model_tr)
# guide side enumeration is not supported
if any(guide_terms["plate_to_step"].values()):
raise NotImplementedError(
"TraceMarkovEnum_ELBO does not yet support guide side Markov enumeration"
)
# build up a lazy expression for the elbo
with funsor.terms.lazy:
# identify and contract out auxiliary variables in the model with partial_sum_product
contracted_factors, uncontracted_factors = [], []
for f in model_terms["log_factors"]:
if model_terms["measure_vars"].intersection(f.inputs):
contracted_factors.append(f)
else:
uncontracted_factors.append(f)
# incorporate the effects of subsampling and handlers.scale through a common scale factor
markov_dims = frozenset({
plate
for plate, step in model_terms["plate_to_step"].items() if step
})
contracted_costs = [
model_terms["scale"] * f
for f in funsor.sum_product.dynamic_partial_sum_product(
funsor.ops.logaddexp,
funsor.ops.add,
model_terms["log_measures"] + contracted_factors,
plate_to_step=model_terms["plate_to_step"],
eliminate=model_terms["measure_vars"] | markov_dims,
)
]
costs = contracted_costs + uncontracted_factors # model costs: logp
costs += [-f for f in guide_terms["log_factors"]
] # guide costs: -logq
# finally, integrate out guide variables in the elbo and all plates
plate_vars = guide_terms["plate_vars"] | model_terms["plate_vars"]
elbo = to_funsor(0, output=funsor.Real)
for cost in costs:
# compute the marginal logq in the guide corresponding to this cost term
log_prob = funsor.sum_product.sum_product(
funsor.ops.logaddexp,
funsor.ops.add,
guide_terms["log_measures"],
plates=plate_vars,
eliminate=(plate_vars | guide_terms["measure_vars"]) -
frozenset(cost.inputs),
)
# compute the expected cost term E_q[logp] or E_q[-logq] using the marginal logq for q
elbo_term = funsor.Integrate(
log_prob, cost,
guide_terms["measure_vars"] & frozenset(cost.inputs))
elbo += elbo_term.reduce(funsor.ops.add,
plate_vars & frozenset(cost.inputs))
# evaluate the elbo, using memoize to share tensor computation where possible
with funsor.interpretations.memoize():
return -to_data(apply_optimizer(elbo))
def _TF2SummaryContext(self):
if FLAGS.disable_tf2_summary:
return contextlib.ExitStack()
return self._tf2_summary_writer.as_default()
def sh(*cmd,
timeout=False,
output_file=None,
input_file=None,
input_text=None,
error=subprocess.STDOUT,
ignore_dry_run=False,
pass_fds=[]):
try:
cmd = list(cmd)
if input_file and input_text:
return 401, "Cannot use both text and file inputs"
# if this is a dry_run, only print the commands that would be ran
if settings.dry_run and not ignore_dry_run:
cmd = "{} cmd: {}".format(os.getcwd(), ' '.join(cmd))
if output_file and not isinstance(output_file, int):
cmd += " > "
cmd += output_file
if error and not isinstance(error, int):
cmd += " 2> "
cmd += error
if input_file and not isinstance(
input_file, int) and os.path.isfile(input_file):
cmd += " < "
cmd += input_file
print(cmd)
return 0, None
with contextlib.ExitStack() as onexit:
# add input redirection if needed
input_file = openfd(input_file, 'r', onexit, True)
# add output redirection if needed
output_file = openfd(output_file, 'w', onexit, False)
# add error redirection if needed
error = openfd(error, 'w', onexit, False)
# run the desired command
# use with statement to make sure proc is cleaned
# don't use subprocess.run because we want to send SIGABRT on exit
with subprocess.Popen(
cmd,
**({
'input': bytes(input_text, encoding='utf-8')
} if input_text else {
'stdin': input_file
}),
stdout=output_file,
stderr=error,
pass_fds=pass_fds) as proc:
try:
out, errout = proc.communicate(
timeout=settings.timeout.single if timeout else None)
return proc.returncode, out.decode(
"latin-1") if out else None, errout.decode(
"latin-1") if errout else None
except subprocess.TimeoutExpired:
if settings.timeout2gdb:
print("Process {} timeout".format(proc.pid))
proc.communicate()
return 124, str(None), "Subprocess Timeout 2 gdb"
else:
proc.send_signal(signal.SIGABRT)
proc.communicate()
return 124, str(None), "Subprocess Timeout 2 gdb"
except Exception as ex:
print("Unexpected error: %s" % ex)
raise
def differentiable_loss(self, model, guide, *args, **kwargs):
# get batched, enumerated, to_funsor-ed traces from the guide and model
with plate(
size=self.num_particles
) if self.num_particles > 1 else contextlib.ExitStack(), enum(
first_available_dim=(-self.max_plate_nesting -
1) if self.max_plate_nesting else None):
guide_tr = trace(guide).get_trace(*args, **kwargs)
model_tr = trace(replay(model, trace=guide_tr)).get_trace(
*args, **kwargs)
# extract from traces all metadata that we will need to compute the elbo
guide_terms = terms_from_trace(guide_tr)
model_terms = terms_from_trace(model_tr)
# build up a lazy expression for the elbo
with funsor.terms.lazy:
# identify and contract out auxiliary variables in the model with partial_sum_product
contracted_factors, uncontracted_factors = [], []
for f in model_terms["log_factors"]:
if model_terms["measure_vars"].intersection(f.inputs):
contracted_factors.append(f)
else:
uncontracted_factors.append(f)
# incorporate the effects of subsampling and handlers.scale through a common scale factor
contracted_costs = [
model_terms["scale"] * f
for f in funsor.sum_product.partial_sum_product(
funsor.ops.logaddexp,
funsor.ops.add,
model_terms["log_measures"] + contracted_factors,
plates=model_terms["plate_vars"],
eliminate=model_terms["measure_vars"],
)
]
# accumulate costs from model (logp) and guide (-logq)
costs = contracted_costs + uncontracted_factors # model costs: logp
costs += [-f for f in guide_terms["log_factors"]
] # guide costs: -logq
# compute expected cost
# Cf. pyro.infer.util.Dice.compute_expectation()
# https://github.com/pyro-ppl/pyro/blob/0.3.0/pyro/infer/util.py#L212
# TODO Replace this with funsor.Expectation
plate_vars = guide_terms["plate_vars"] | model_terms["plate_vars"]
# compute the marginal logq in the guide corresponding to each cost term
targets = dict()
for cost in costs:
input_vars = frozenset(cost.inputs)
if input_vars not in targets:
targets[input_vars] = funsor.Tensor(
funsor.ops.new_zeros(
funsor.tensor.get_default_prototype(),
tuple(v.size for v in cost.inputs.values()),
),
cost.inputs,
cost.dtype,
)
with AdjointTape() as tape:
logzq = funsor.sum_product.sum_product(
funsor.ops.logaddexp,
funsor.ops.add,
guide_terms["log_measures"] + list(targets.values()),
plates=plate_vars,
eliminate=(plate_vars | guide_terms["measure_vars"]),
)
marginals = tape.adjoint(funsor.ops.logaddexp, funsor.ops.add,
logzq, tuple(targets.values()))
# finally, integrate out guide variables in the elbo and all plates
elbo = to_funsor(0, output=funsor.Real)
for cost in costs:
target = targets[frozenset(cost.inputs)]
logzq_local = marginals[target].reduce(
funsor.ops.logaddexp,
frozenset(cost.inputs) - plate_vars)
log_prob = marginals[target] - logzq_local
elbo_term = funsor.Integrate(
log_prob,
cost,
guide_terms["measure_vars"] & frozenset(log_prob.inputs),
)
elbo += elbo_term.reduce(funsor.ops.add,
plate_vars & frozenset(cost.inputs))
# evaluate the elbo, using memoize to share tensor computation where possible
with funsor.interpretations.memoize():
return -to_data(apply_optimizer(elbo))
def main():
import argparse
formats = loading.get_formats()
parser = argparse.ArgumentParser(
formatter_class=type(
"_HelpFormatter",
(argparse.ArgumentDefaultsHelpFormatter, argparse.RawTextHelpFormatter),
{},
)
)
parser.print_usage = parser.print_help # hack
parser.add_argument(
"--log",
choices=list(logging._nameToLevel.keys()),
default="INFO",
dest="log_level",
help="-",
)
parser.add_argument("-q", "--quiet", action="store_true", help="-")
parser.add_argument("--debug", action="store_true", help="-")
subparsers = parser.add_subparsers(dest="subcommand", title="subcommands")
subparsers.required = True
# merge
fn = merge
sparser = subparsers.add_parser(
fn.__name__, help=fn.__doc__, formatter_class=parser.formatter_class
)
sparser.set_defaults(subcommand=fn)
sparser.add_argument("files", nargs="*", default=None, help="-")
sparser.add_argument("--dst", default=None, help="-")
sparser.add_argument("--strict", action="store_true", help="-")
sparser.add_argument("--style", default="ref", choices=["ref", "whole"], help="-")
sparser.add_argument("--wrap", default=None, help="-")
sparser.add_argument("--wrap-section", default="definitions", help="-")
# tojsonschema
fn = tojsonschema
sparser = subparsers.add_parser(
fn.__name__, help=fn.__doc__, formatter_class=parser.formatter_class
)
sparser.set_defaults(subcommand=fn)
sparser.add_argument("--src", default=None, help="-")
sparser.add_argument("--dst", default=None, help="-")
sparser.add_argument("--name", default="top", help="-")
# json2swagger
fn = json2swagger
sparser = subparsers.add_parser(
fn.__name__, help=fn.__doc__, formatter_class=parser.formatter_class
)
sparser.set_defaults(subcommand=fn)
sparser.add_argument("files", nargs="*", default=None, help="-")
sparser.add_argument("--dst", default=None, help="-")
sparser.add_argument(
"-o", "--output-format", default=None, choices=formats, help="-"
)
sparser.add_argument("--name", default="top", help="-")
sparser.add_argument("--detector", default="Detector", help="-")
sparser.add_argument("--emitter", default="Emitter", help="-")
sparser.add_argument("--annotate", default=None, help="-")
sparser.add_argument(
"--emit", default="schema", choices=["schema", "info"], help="-"
)
sparser.add_argument("--with-minimap", action="store_true", help="-")
sparser.add_argument("--without-example", action="store_true", help="-")
# flatten
fn = flatten
sparser = subparsers.add_parser(
fn.__name__, help=fn.__doc__, formatter_class=parser.formatter_class
)
sparser.set_defaults(subcommand=fn)
sparser.add_argument("src", nargs="?", default=None, help="-")
sparser.add_argument("--dst", default=None, help="-")
sparser.add_argument(
"-i", "--input-format", default=None, choices=formats, help="-"
)
sparser.add_argument(
"-o", "--output-format", default=None, choices=formats, help="-"
)
sparser.add_argument("-f", "--format", default=None, choices=formats, help="-")
args = parser.parse_args()
with contextlib.ExitStack() as s:
params = vars(args)
if params.pop("quiet"):
args.log_level = logging._levelToName[logging.WARNING]
s.enter_context(warnings.catch_warnings())
warnings.simplefilter("ignore")
logging.basicConfig(level=getattr(logging, params.pop("log_level")))
with traceback_shortly(params.pop("debug")):
return params.pop("subcommand")(**params)
def setUp(self):
self.fixtures = contextlib.ExitStack()
self.addCleanup(self.fixtures.close)
def setUp(self):
self.fixtures = contextlib.ExitStack()
self.addCleanup(self.fixtures.close)
self.fixtures.enter_context(tempdir_as_cwd())
build_files(self.files)
def train(
rank: int,
world_size: int,
*,
resume: bool,
location: str,
name: str,
quiet: bool,
save_every: int,
validate_every: int,
post_command: str,
explicit_args: set[str],
params: Hyperparams,
):
# Get filepath within path context
fpath = lambda path: os.path.join(location, path) if isinstance(path, str) else os.path.join(location, *path)
# Setup multi-gpu if used
setup(rank, world_size)
is_master = rank < 1 # Are we on the main node?
is_distributed = rank != -1 # Are we performing distributed computing?
num_workers = torch.distributed.get_world_size() if is_distributed else 1
# Update locations
TrainResults.subfolder = name
Hyperparams.subfolder = name
# Setup logger
log.configure(
os.path.join(location, name, "pretraining-worker=%s.log" % (rank if is_distributed else 0)),
"DaLUKE pretraining on node %i" % rank,
log_commit = True,
print_level = (Levels.INFO if quiet else Levels.DEBUG) if is_master else None,
append = resume, # Append to existing log file if we are resuming training
)
post_time, post_command = parse_post_command(post_command)
execute_post_command = False
if post_time:
log("Quitting in %.2f h and running command '%s'" % ((post_time-time.time())/3600, post_command))
if resume:
log("Resuming from %s" % name)
# Load results and hyperparameters from earlier training
res = TrainResults.load(location)
# Close unended profiles
close_tt(res.tt)
TT.fuse(res.tt)
res.tt = TT
tmp_saved_pu = res.parameter_update
loaded_params = Hyperparams.load(location)
# Overwrite ff-size if given explicitly
if "ff_size" in explicit_args:
loaded_params.ff_size = params.ff_size
params = loaded_params
else:
tmp_saved_pu = None
log.section("Starting pretraining with the following hyperparameters", params)
log("Training using %i workers" % num_workers)
log("Reading metadata and entity vocabulary")
with open(fpath(DatasetBuilder.metadata_file)) as f:
metadata = json.load(f)
with open(fpath(DatasetBuilder.entity_vocab_file)) as f:
entity_vocab = json.load(f)
log("Loaded metadata:", json.dumps(metadata, indent=4))
log(f"Loaded entity vocabulary of {len(entity_vocab)} entities")
if params.ent_min_mention:
log("Removing entities with less than %i mentions" % params.ent_min_mention)
entity_vocab = { ent: info for ent, info in entity_vocab.items()
if info["count"] >= params.ent_min_mention or ent in {"[PAD]", "[UNK]", "[MASK]"} }
log("After filtering, entity vocab now has %i entities" % len(entity_vocab))
# Device should be cuda:rank or just cuda if single gpu, else cpu
if is_distributed:
device = torch.device("cuda", index=rank)
else:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
log.debug(
"Hardware for this worker:",
"CPU: %s" % cpuinfo.get_cpu_info()["brand_raw"],
"GPU: %s" % (torch.cuda.get_device_name(device) if torch.cuda.is_available() else "NA"),
sep="\t\n",
)
if params.entity_loss_weight:
log("Setting up loss function with entity loss weighting")
# Don't weigh special tokens
weights = torch.Tensor([0, 0, 0, *(1 / info["count"] for info in entity_vocab.values() if info["count"])]).to(device)
entity_criterion = nn.CrossEntropyLoss(weight=weights)
else:
log("Setting up loss function without entity loss weighting")
entity_criterion = nn.CrossEntropyLoss()
word_criterion = nn.CrossEntropyLoss()
loss_calculator = lambda w, e: params.word_ent_weight * w + (1 - params.word_ent_weight) * e
# Load dataset and training results
bert_config = AutoConfig.from_pretrained(metadata["base-model"])
if metadata["reduced-vocab"]:
token_map_file = fpath(DatasetBuilder.token_map_file)
log("Loading token map from '%s'" % token_map_file)
token_map = np.load(token_map_file)
tokenizer = AutoTokenizer.from_pretrained(metadata["base-model"])
*__, unk_id = get_special_ids(tokenizer)
token_reduction = token_map_to_token_reduction(token_map, unk_id)
else:
token_map = None
log("Building dataset")
data = DataLoader(
location,
metadata,
entity_vocab,
device,
params.word_mask_prob,
params.word_unmask_prob,
params.word_randword_prob,
params.ent_mask_prob,
vocab_size=metadata["vocab-size"],
token_map=token_map,
ent_min_mention=params.ent_min_mention,
)
sampler = (DistributedSampler if is_distributed else RandomSampler)(data.train_examples)
log("Built %i examples" % len(data))
loader = data.get_dataloader(params.ff_size, sampler)
val_loader = data.get_dataloader(params.ff_size, SequentialSampler(data.val_examples), validation=True)
# Number of subbatches in each parameter update (batch)
grad_accumulation_steps = params.batch_size // (params.ff_size * num_workers)
# How many full batches can be made from the dataset
batches_in_data = len(data) // params.batch_size
log(
"Parameter updates: %i" % params.parameter_updates,
"Subbatches per parameter update: %i" % grad_accumulation_steps,
"Subbatches generated: %i" % len(loader),
"Batches needed to cover dataset: %i" % batches_in_data,
)
if not resume:
# Calculate parameter differences, when at least 20k examples have been seen
paramdiff_every = ceil(MIN_EXAMPLES_PER_PARAMDIFF / params.batch_size)
log("Recalculating parameter differences every %i'th parameter update" % paramdiff_every)
top_k = [1, 3, 10]
log("Calculating top %s accuracies" % top_k)
if validate_every:
val_updates = unique(np.array(
np.arange(-1, params.parameter_updates, validate_every).tolist() + [params.parameter_updates-1]
))[1:]
else:
val_updates = np.array([], dtype=int)
res = TrainResults(
runtime = np.zeros(params.parameter_updates),
lr = np.zeros(params.parameter_updates),
parameter_update = 0,
losses = np.zeros(params.parameter_updates),
scaled_loss = np.zeros(params.parameter_updates),
top_k = top_k,
w_losses = np.zeros(params.parameter_updates),
e_losses = np.zeros(params.parameter_updates),
w_accuracies = np.zeros((params.parameter_updates, len(top_k))),
e_accuracies = np.zeros((params.parameter_updates, len(top_k))),
val_param_updates = val_updates,
val_losses = np.zeros(len(val_updates)),
val_w_losses = np.zeros(len(val_updates)),
val_e_losses = np.zeros(len(val_updates)),
val_w_accuracies = np.zeros((len(val_updates), len(top_k))),
val_e_accuracies = np.zeros((len(val_updates), len(top_k))),
paramdiff_every = paramdiff_every,
groups_to_slices = None, # Set later
orig_params = None,
paramdiff_1 = None,
luke_exclusive_params = None, # Set later
att_mats_from_base = None, # Set later
tt = TT,
)
save_pus = set(range(-1, params.parameter_updates, save_every)).union({params.parameter_updates-1})
log("Saving model at parameter updates: %s" % sorted(save_pus),
"Validating at parameter updates: %s" % res.val_param_updates.tolist())
# Build model, possibly by loading previous weights
log.section("Setting up model")
bert_config = AutoConfig.from_pretrained(metadata["base-model"])
if params.ent_hidden_size is None:
params.ent_hidden_size = bert_config.hidden_size
else:
assert params.ent_hidden_size <= bert_config.hidden_size,\
"Entity hidden size (%i) cannot be larger than hidden size in '%s' (%i)" % (
params.hidden_size,
metadata["base-model"],
bert_config.hidden_size,
)
log("Initializing model")
model_cls = BertAttentionPretrainTaskDaLUKE if params.bert_attention else PretrainTaskDaLUKE
model = model_cls(
bert_config,
ent_vocab_size = len(entity_vocab),
ent_embed_size = params.ent_embed_size,
ent_hidden_size = params.ent_hidden_size,
ent_intermediate_size = params.ent_intermediate_size,
).to(device)
bert_config.vocab_size = metadata["vocab-size"]
log("Bert config", bert_config.to_json_string())
if params.lukeinit:
log("Initializing weights in accordance with LUKE")
model.apply(lambda module: model.init_weights(module, bert_config.initializer_range))
# Load parameters from base model
if not params.no_base_model:
log("Loading base model parameters")
with TT.profile("Loading base model parameters"):
base_model = AutoModelForPreTraining.from_pretrained(metadata["base-model"])
new_weights = load_base_model_weights(
model,
base_model.state_dict(),
params.bert_attention,
)
if metadata["reduced-vocab"]:
log("Removing unneeded token weights")
reduced_model = model_cls(
bert_config,
ent_vocab_size = len(entity_vocab),
ent_embed_size = params.ent_embed_size,
ent_hidden_size = params.ent_hidden_size,
ent_intermediate_size = params.ent_intermediate_size,
).to(device)
copy_with_reduced_state_dict(token_reduction, model, reduced_model)
model = reduced_model
else:
new_weights = set(model.state_dict())
# Initialize self-attention query matrices to BERT word query matrices
att_mat_keys = set()
if not params.bert_attention and not params.no_base_model:
log("Initializing new attention matrices with%s PCA" % ("" if params.pcainit else "out"))
att_mat_keys = model.init_special_attention(params.pcainit, device)
if not resume:
res.luke_exclusive_params = new_weights
res.att_mats_from_base = att_mat_keys
if is_master:
res.orig_params = all_params(model).cpu().numpy()
log("Pretraining model initialized with %s parameters" % thousand_seps(len(model)))
# Unfixes params at this parameter update
unfix_base_model_params_pu = round(params.bert_fix_prop * params.parameter_updates)
log("Unfixing base model params after %i parameter updates" % unfix_base_model_params_pu)
if resume:
mpath = fpath((TrainResults.subfolder, MODEL_OUT.format(i=res.parameter_update)))
log("Loading model from '%s'" % mpath)
model.load_state_dict(torch.load(mpath, map_location=device))
log(f"Resuming training saved at parameter update {res.parameter_update}")
else:
res.groups_to_slices, t = all_params_groups_to_slices(model, bert_config.num_hidden_layers)
log("Parameter groups and positions", t)
res.paramdiff_1 = { name: np.zeros(ceil(params.parameter_updates/res.paramdiff_every)) for name in res.groups_to_slices }
if is_distributed:
model = DDP(model, device_ids=[rank], find_unused_parameters=True)
non_ddp_model = model.module if is_distributed else model
log("Setting up optimizer, scaler, and learning rate scheduler")
optimizer = get_optimizer(non_ddp_model, params.weight_decay, params.lr)
scaler = amp.GradScaler() if params.fp16 else None
scheduler = get_lr_scheduler(
optimizer,
int(params.warmup_prop * params.parameter_updates),
params.parameter_updates,
unfix_base_model_params_pu,
)
if resume:
optimizer.load_state_dict(torch.load(fpath((TrainResults.subfolder, OPTIMIZER_OUT.format(i=res.parameter_update))), map_location=device))
scheduler.load_state_dict(torch.load(fpath((TrainResults.subfolder, SCHEDULER_OUT.format(i=res.parameter_update))), map_location=device))
if params.fp16:
scaler.load_state_dict(torch.load(fpath((TrainResults.subfolder, SCALER_OUT.format(i=res.parameter_update))), map_location=device))
res.parameter_update += 1 # We saved the data at pu i, but should now commence pu i+1
log.debug("Time distribution before starting training", TT)
log_memory_stats(device)
log.section(f"Training DaLUKE for {params.parameter_updates} parameter updates")
model.zero_grad() # To avoid tracking of model parameter manipulation
model.train()
# Start with transfer learned weights locked
fix_base_model_params(res.luke_exclusive_params, non_ddp_model, True)
fixed_params = True
# Save initial parameters
if is_master and not resume:
with TT.profile("Saving progress"):
paths = save_training(location, params, model.module if is_distributed else model,
res, optimizer, scheduler, scaler, -1)
log.debug("Saved initial state to", *paths)
batch_iter = iter(loader)
for i in range(res.parameter_update, params.parameter_updates):
TT.profile("Parameter update")
res.parameter_update = i
if i >= unfix_base_model_params_pu and fixed_params:
log("Unfixing base model params")
fix_base_model_params(res.luke_exclusive_params, model, False)
fixed_params = False
if is_distributed and i % batches_in_data == 0:
sampler.set_epoch(i // batches_in_data)
# Losses and accuracies for this parameter update
t_loss, w_loss, e_loss, s_loss = 0, 0, 0, 0
w_accuracies = np.zeros((grad_accumulation_steps, len(res.top_k)))
e_accuracies = np.zeros((grad_accumulation_steps, len(res.top_k)))
# Loop over enough batches to make a parameter update
for j in range(grad_accumulation_steps):
TT.profile("Sub-batch")
try:
batch = next(batch_iter)
except StopIteration:
batch_iter = iter(loader)
batch = next(batch_iter)
TT.profile("FP and gradients")
with amp.autocast() if params.fp16 else contextlib.ExitStack():
word_preds, ent_preds = model(batch)
# Compute and backpropagate loss
word_loss = word_criterion(word_preds, batch.word_mask_labels)
ent_loss = entity_criterion(ent_preds, batch.ent_mask_labels)
has_entities = not torch.isnan(ent_loss).item()
ent_loss = torch.nan_to_num(ent_loss)
loss = loss_calculator(word_loss, ent_loss)
loss /= grad_accumulation_steps
# Only sync parameters on grad updates, aka last pass of this loop
with model.no_sync() if is_distributed and j < grad_accumulation_steps - 1 else contextlib.ExitStack():
if params.fp16:
scaled_loss = scaler.scale(loss)
scaled_loss.backward()
s_loss += scaled_loss.item()
else:
loss.backward()
t_loss += loss.item()
w_loss += word_loss.item() / grad_accumulation_steps
e_loss += ent_loss.item() / grad_accumulation_steps if has_entities else 0
if torch.cuda.is_available():
torch.cuda.synchronize(rank if is_distributed else None)
TT.end_profile()
# Save accuracy for statistics
if is_master:
with TT.profile("Training accuracy"):
w_accuracies[j] = top_k_accuracy(batch.word_mask_labels, word_preds, res.top_k)
e_accuracies[j] = top_k_accuracy(batch.ent_mask_labels, ent_preds, res.top_k)
TT.end_profile()
# Update model parameters
with TT.profile("Parameter step"):
if params.fp16:
scaler.step(optimizer)
scaler.update()
else:
optimizer.step()
scheduler.step()
model.zero_grad()
# Calculate how much gradient has changed
if is_master and i % res.paramdiff_every == 0:
with torch.no_grad(), TT.profile("Parameter changes"):
log.debug("Calculating parameter changes")
orig_pars = torch.from_numpy(res.orig_params).to(device)
current_pars = all_params(model.module if is_distributed else model)
absdiff = torch.abs(current_pars-orig_pars)
for blockname, slice_ in res.groups_to_slices.items():
j = i // res.paramdiff_every
res.paramdiff_1[blockname][j] = absdiff[slice_].sum().item()
del orig_pars, current_pars
res.losses[i] = t_loss
res.w_losses[i] = w_loss
res.e_losses[i] = e_loss
res.scaled_loss[i] = s_loss
res.lr[i] = scheduler.get_last_lr()[0]
res.w_accuracies[i] = np.mean(w_accuracies, axis=0)
res.e_accuracies[i] = np.nanmean(e_accuracies, axis=0)
res.runtime[i] = TT.end_profile()
log.debug(
"Performed parameter update %i / %i in %.2f s" % (i, params.parameter_updates-1, res.runtime[i]),
f" Loss (total, word, entity, scaled): {t_loss:9.4f}, {w_loss:9.4f}, {e_loss:9.4f}, {s_loss:.4f}",
f" Accuracy (word, entity): {100*res.w_accuracies[i, 0]:7.2f} %, {100*res.e_accuracies[i, 0]:7.2f} %",
)
if i in res.val_param_updates and is_master:
TT.profile("Model validation")
log("Validating model")
vi = res.val_param_updates.tolist().index(i)
res.val_w_losses[vi], res.val_e_losses[vi], res.val_w_accuracies[vi], res.val_e_accuracies[vi] =\
validate_model(model, val_loader, word_criterion, entity_criterion, res.top_k)
res.val_losses[vi] = loss_calculator(res.val_w_losses[vi], res.val_e_losses[vi])
log(
"Validation loss:",
" Total: %9.4f" % res.val_losses[vi],
" Word: %9.4f" % res.val_w_losses[vi],
" Entity: %9.4f" % res.val_e_losses[vi],
"Validation accuracy:",
" Word: %7.2f %%" % (100 * res.val_w_accuracies[vi, 0]),
" Entity: %7.2f %%" % (100 * res.val_e_accuracies[vi, 0]),
)
model.train()
TT.end_profile()
log.debug("Time distribution so far", TT)
# Save results and model
if is_master and i in save_pus:
with TT.profile("Saving progress"):
save_progress(location, i, tmp_saved_pu, save_pus, params,
model.module if is_distributed else model, res, optimizer, scheduler, scaler)
if i in save_pus:
log_memory_stats(device)
# If timed out, save, quit, and run resume command
if post_time and time.time() > post_time:
log_memory_stats(device)
log.section("Time limit reached. Quitting and running command '%s'" % post_command)
with TT.profile("Saving progress"):
save_progress(location, i, tmp_saved_pu, save_pus, params,
model.module if is_distributed else model, res, optimizer, scheduler, scaler)
execute_post_command = True
break
log.debug("Time distribution", TT)
# Clean up multi-gpu if used
cleanup(rank)
if is_master and execute_post_command:
os.system(post_command)
def _load_data(args, s):
# Prepare the simulation parameters
if args.min_maf is not None or args.max_maf is not None:
if args.min_maf is None:
args.min_maf = 0.01
if args.max_maf is None:
args.max_maf = 0.05
s.sample_mafs(args.min_maf, args.max_maf)
if args.annotation_vector is not None:
logger.debug('Loading annotation vector')
a = numpy.loadtxt(args.annotation_vector).astype('int8')
if a.shape[0] != args.num_variants:
raise _A('{} variants present in annotations file, but {} specified'.format(a.shape[0], args.num_variants))
s.load_annotations(a)
elif args.annotation_matrix is not None:
logger.debug('Loading annotation matrix')
with gzip.open(args.annotation_matrix, 'rt') as f:
a = numpy.loadtxt(f).astype('int8')
if a.shape[0] != args.num_variants:
raise _A('{} variants present in annotations file, but {} specified'.format(a.shape[0], args.num_variants))
if args.annotation_matrix_column < 0:
raise _A('Annotation column must be non-negative')
if args.annotation_matrix_column > a.shape[1]:
raise _A('{} columns present in annotation matrix, but column {} specified'.format(a.shape[1], args.annotation_matrix_column))
s.load_annotations(a, args.annotation_matrix_column)
if args.permute_causal:
logger.debug('Generating effects with permuted causal indicator')
s.sample_effects(pve=args.pve, annotation_params=args.annotation, permute=True)
# Load/generate the genotypes and phenotypes
if args.load_oxstats:
logger.debug('Loading OXSTATS datasets')
with contextlib.ExitStack() as stack:
data = [stack.enter_context(ctra.formats.oxstats_genotypes(*a))
for a in ctra.algorithms.kwise(args.load_oxstats, 2)]
samples = list(itertools.chain.from_iterable(s for _, _, s, _ in data))
merged = ctra.formats.merge_oxstats([d for _, _, _, d in data])
probs = ([float(x) for x in row[5:]] for row in merged)
if args.num_samples > len(samples):
logger.error('{} individuals present in OXSTATS data, but {} were specified'.format(len(samples), args.num_samples))
sys.exit(1)
x = numpy.array(list(itertools.islice(probs, args.num_variants)))
p, n = x.shape
if p < args.num_variants:
logger.error('{} variants present in OXSTATS data, but {} were specified'.format(p, args.num_variants))
sys.exit(1)
n = n // 3
x = (x.reshape(p, -1, 3) * numpy.array([0, 1, 2])).sum(axis=2).T[:n,:]
s.estimate_mafs(x)
y = s.compute_liabilities(x)
elif args.load_hdf5:
with h5py.File(args.load_hdf5) as f:
logger.debug('Loading HDF5 dataset')
x = f['dosage'][:args.num_samples, :args.num_variants]
logger.debug('Re-computing liabilities')
s.estimate_mafs(x)
y = s.compute_liabilities(x)
else:
if args.model == 'logistic':
x, y = s.sample_case_control(n=args.num_samples, K=args.prevalence, P=args.study_prop)
else:
x, y = s.sample_gaussian(n=args.num_samples)
# Hold out samples
if args.model == 'logistic':
# Randomly subsample hold out set
validation = numpy.zeros(args.num_samples, dtype='bool')
hold_out = s.random.choice(args.num_samples, args.validation, replace=False)
validation[hold_out] = True
x_validate = x[validation]
y_validate = y[validation]
# Permute the training set so minibatches are balanced in expectation
perm = s.random.permutation(args.num_samples - args.validation)
x = x[~validation][perm]
y = y[~validation][perm]
else:
# Assume samples are exchangeable
x_validate = x[-args.validation:]
y_validate = y[-args.validation:]
x = x[:-args.validation]
y = y[:-args.validation]
x -= x.mean(axis=0)
x_validate -= x_validate.mean(axis=0)
if args.model != 'logistic':
y -= y.mean()
y_validate -= y_validate.mean()
return x, y, x_validate, y_validate
def prune_state_dict(state_dict, model_cfg: Optional[DictConfig]):
"""Prune the given state_dict if desired for LayerDrop
(https://arxiv.org/abs/1909.11556).
Training with LayerDrop allows models to be robust to pruning at inference
time. This function prunes state_dict to allow smaller models to be loaded
from a larger model and re-maps the existing state_dict for this to occur.
It's called by functions that load models from checkpoints and does not
need to be called directly.
"""
arch = None
if model_cfg is not None:
arch = (model_cfg._name if isinstance(model_cfg, DictConfig) else
getattr(model_cfg, "arch", None))
if not model_cfg or arch is None or arch == "ptt_transformer":
# args should not be none, but don't crash if it is.
return state_dict
encoder_layers_to_keep = getattr(model_cfg, "encoder_layers_to_keep", None)
decoder_layers_to_keep = getattr(model_cfg, "decoder_layers_to_keep", None)
if not encoder_layers_to_keep and not decoder_layers_to_keep:
return state_dict
# apply pruning
logger.info(
"Pruning model to specified layer configuration - this works best if the model was trained with LayerDrop"
)
def create_pruning_pass(layers_to_keep, layer_name):
keep_layers = sorted(
int(layer_string) for layer_string in layers_to_keep.split(","))
mapping_dict = {}
for i in range(len(keep_layers)):
mapping_dict[str(keep_layers[i])] = str(i)
regex = re.compile(
r"^{layer}.*\.layers\.(\d+)".format(layer=layer_name))
return {"substitution_regex": regex, "mapping_dict": mapping_dict}
pruning_passes = []
if encoder_layers_to_keep:
pruning_passes.append(
create_pruning_pass(encoder_layers_to_keep, "encoder"))
if decoder_layers_to_keep:
pruning_passes.append(
create_pruning_pass(decoder_layers_to_keep, "decoder"))
new_state_dict = {}
for layer_name in state_dict.keys():
match = re.search(r"\.layers\.(\d+)\.", layer_name)
# if layer has no number in it, it is a supporting layer, such as an
# embedding
if not match:
new_state_dict[layer_name] = state_dict[layer_name]
continue
# otherwise, layer should be pruned.
original_layer_number = match.group(1)
# figure out which mapping dict to replace from
for pruning_pass in pruning_passes:
if original_layer_number in pruning_pass[
"mapping_dict"] and pruning_pass[
"substitution_regex"].search(layer_name):
new_layer_number = pruning_pass["mapping_dict"][
original_layer_number]
substitution_match = pruning_pass["substitution_regex"].search(
layer_name)
new_state_key = (layer_name[:substitution_match.start(1)] +
new_layer_number +
layer_name[substitution_match.end(1):])
new_state_dict[new_state_key] = state_dict[layer_name]
# Since layers are now pruned, *_layers_to_keep are no longer needed.
# This is more of "It would make it work fix" rather than a proper fix.
if isinstance(model_cfg, DictConfig):
context = open_dict(model_cfg)
else:
context = contextlib.ExitStack()
with context:
if hasattr(model_cfg, "encoder_layers_to_keep"):
model_cfg.encoder_layers_to_keep = None
if hasattr(model_cfg, "decoder_layers_to_keep"):
model_cfg.decoder_layers_to_keep = None
return new_state_dict
def decorated(self, **kwargs):
"""A wrapped test method that can treat some arguments in a special way."""
original_kwargs = kwargs.copy()
# Skip combinations that are going to be executed in a different testing
# environment.
reasons_to_skip = []
for combination in test_combinations:
should_execute, reason = combination.should_execute_combination(
original_kwargs.copy())
if not should_execute:
reasons_to_skip.append(' - ' + reason)
if reasons_to_skip:
self.skipTest('\n'.join(reasons_to_skip))
customized_parameters = []
for combination in test_combinations:
customized_parameters.extend(combination.parameter_modifiers())
customized_parameters = set(customized_parameters)
# The function for running the test under the total set of
# `context_managers`:
def execute_test_method():
requested_parameters = tf_inspect.getfullargspec(test_method).args
for customized_parameter in customized_parameters:
for argument, value in customized_parameter.modified_arguments(
original_kwargs.copy(), requested_parameters).items():
if value is ParameterModifier.DO_NOT_PASS_TO_THE_TEST:
kwargs.pop(argument, None)
else:
kwargs[argument] = value
omitted_arguments = set(requested_parameters).difference(
set(list(kwargs.keys()) + ['self']))
if omitted_arguments:
raise ValueError('The test requires parameters whose arguments '
'were not passed: {} .'.format(omitted_arguments))
missing_arguments = set(list(kwargs.keys()) + ['self']).difference(
set(requested_parameters))
if missing_arguments:
raise ValueError('The test does not take parameters that were passed '
': {} .'.format(missing_arguments))
kwargs_to_pass = {}
for parameter in requested_parameters:
if parameter == 'self':
kwargs_to_pass[parameter] = self
else:
kwargs_to_pass[parameter] = kwargs[parameter]
with self.cached_session():
test_method(**kwargs_to_pass)
# Install `context_managers` before running the test:
context_managers = []
for combination in test_combinations:
for manager in combination.context_managers(
original_kwargs.copy()):
context_managers.append(manager)
if hasattr(contextlib, 'nested'): # Python 2
# TODO(isaprykin): Switch to ExitStack when contextlib2 is available.
with contextlib.nested(*context_managers):
execute_test_method()
else: # Python 3
with contextlib.ExitStack() as context_stack:
for manager in context_managers:
context_stack.enter_context(manager)
execute_test_method()
''' Main class, prepare kafka and citisim broker and connects them '''
def run(self, libcitisim_config, producers):
''' Main function '''
citisim_broker = libcitisim.Broker(libcitisim_config)
for topic_name, producer in producers.items():
print("Subscribing to '{}' topic".format(topic_name))
citisim_broker.subscribe(topic_name,
EventMirrorVisitor(producer).callback)
print("Awaiting data...")
citisim_broker.wait_for_events()
if __name__ == "__main__":
args = SubscriberArgumentParser().parse_args(sys.argv[1:])
kafka_events_handler = pykafka.handlers.ThreadingHandler()
kafka_cluster = pykafka.cluster.Cluster(hosts=args.kafka_cluster,
handler=kafka_events_handler)
with contextlib.ExitStack() as exit_stack:
kafka_producers = {
topic_name: exit_stack.enter_context(
kafka_mirror_producer(kafka_cluster, topic_name))
for topic_name in args.forward_topic
}
SubscriberMirror().run(libcitisim_config=args.libcitisim_args[0],
producers=kafka_producers)
def run(self, num_frames, preview_alpha, image_format, image_folder,
enable_streaming):
logger.info('Starting...')
leds = Leds()
with contextlib.ExitStack() as stack:
player = stack.enter_context(Player(gpio=BUZZER_GPIO, bpm=10))
photographer = stack.enter_context(
Photographer(image_format, image_folder))
animator = stack.enter_context(Animator(leds))
# Forced sensor mode, 1640x1232, full FoV. See:
# https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes
# This is the resolution inference run on.
# Use half of that for video streaming (820x616).
camera = stack.enter_context(
PiCamera(sensor_mode=4, resolution=(820, 616)))
stack.enter_context(PrivacyLed(leds))
server = None
if enable_streaming:
server = stack.enter_context(StreamingServer(camera))
server.run()
def take_photo():
logger.info('Button pressed.')
player.play(BEEP_SOUND)
photographer.shoot(camera)
if preview_alpha > 0:
camera.start_preview(alpha=preview_alpha)
button = Button(BUTTON_GPIO)
button.when_pressed = take_photo
joy_score_moving_average = MovingAverage(5) #Changed it from 10
prev_joy_score = 0.0
with CameraInference(face_detection.model()) as inference:
logger.info('Model loaded.')
player.play(MODEL_LOAD_SOUND)
for i, result in enumerate(inference.run()):
faces = face_detection.get_faces(result)
photographer.update_faces(faces)
joy_score = joy_score_moving_average.next(
average_joy_score(faces))
animator.update_joy_score(joy_score)
if server:
data = server_inference_data(result.width,
result.height, faces,
joy_score)
server.send_inference_data(data)
if joy_score > JOY_SCORE_PEAK > prev_joy_score:
player.play(JOY_SOUND)
## picoSpeakNow(list_happy[np.random.randint(0,N_HAPPY)])
## os.system('pico2wave -w test.wav "keep smiling. I feed off of smile energy... do not let the smile die down." && aplay test.wav')
## time.sleep(3)
espeak_happy = 'espeak -s160 -g6 -ven+f3 ' + '"' + if_happy_list[
np.random.randint(0, N_HAPPY)] + '"'
os.system(espeak_happy)
elif joy_score < 0.35 < prev_joy_score:
player.play(SAD_SOUND)
espeak_sad = 'espeak -s160 -g6 -ven+f3 ' + '"' + if_sad[
0] + '"'
os.system(espeak_sad)
## picoSpeakNow(list_sad[np.random.randint(0,N_SAD)])
## time.sleep(3)
## os.system('espeak "Keep smiling. I feed off of smile energy... do not let the smile die down"')
## os.system('pico2wave -w test.wav "start smiling. I feed off of smile energy... do not let the smile die down." && aplay test.wav')
prev_joy_score = joy_score
if self._done.is_set() or i == num_frames:
break
def test_enable_configures_apt_sources_and_auth_files(
self, _m_get_cloud_type, m_setup_apt_proxy, entitlement):
"""When entitled, configure apt repo auth token, pinning and url."""
patched_packages = ["a", "b"]
expected_conditional_packages = [
"openssh-server",
"openssh-server-hmac",
"strongswan",
"strongswan-hmac",
]
with contextlib.ExitStack() as stack:
m_add_apt = stack.enter_context(
mock.patch("uaclient.apt.add_auth_apt_repo"))
m_add_pinning = stack.enter_context(
mock.patch("uaclient.apt.add_ppa_pinning"))
m_installed_pkgs = stack.enter_context(
mock.patch(
"uaclient.apt.get_installed_packages",
return_value=["openssh-server", "strongswan"],
))
m_subp = stack.enter_context(
mock.patch("uaclient.util.subp", return_value=("", "")))
m_can_enable = stack.enter_context(
mock.patch.object(entitlement, "can_enable"))
stack.enter_context(
mock.patch("uaclient.util.handle_message_operations"))
stack.enter_context(
mock.patch(M_GETPLATFORM, return_value={"series": "xenial"}))
stack.enter_context(mock.patch(M_REPOPATH + "os.path.exists"))
# Note that this patch uses a PropertyMock and happens on the
# entitlement's type because packages is a property
m_packages = mock.PropertyMock(return_value=patched_packages)
stack.enter_context(
mock.patch.object(type(entitlement), "packages", m_packages))
stack.enter_context(
mock.patch("uaclient.util.is_container", return_value=False))
m_can_enable.return_value = (True, None)
assert (True, None) == entitlement.enable()
repo_url = "http://{}".format(entitlement.name.upper())
add_apt_calls = [
mock.call(
"/etc/apt/sources.list.d/ubuntu-{}.list".format(
entitlement.name),
repo_url,
"{}-token".format(entitlement.name),
["xenial"],
entitlement.repo_key_file,
)
]
apt_pinning_calls = [
mock.call(
"/etc/apt/preferences.d/ubuntu-{}".format(entitlement.name),
repo_url,
entitlement.origin,
1001,
)
]
install_cmd = []
install_cmd.append(
mock.call(
[
"apt-get",
"install",
"--assume-yes",
"--allow-downgrades",
'-o Dpkg::Options::="--force-confdef"',
'-o Dpkg::Options::="--force-confold"',
] + patched_packages,
capture=True,
retry_sleeps=apt.APT_RETRIES,
env={"DEBIAN_FRONTEND": "noninteractive"},
))
for pkg in expected_conditional_packages:
install_cmd.append(
mock.call(
[
"apt-get",
"install",
"--assume-yes",
"--allow-downgrades",
'-o Dpkg::Options::="--force-confdef"',
'-o Dpkg::Options::="--force-confold"',
pkg,
],
capture=True,
retry_sleeps=apt.APT_RETRIES,
env={"DEBIAN_FRONTEND": "noninteractive"},
))
subp_calls = [
mock.call(
["apt-mark", "showholds"],
capture=True,
retry_sleeps=apt.APT_RETRIES,
env={},
),
mock.call(
["apt-get", "update"],
capture=True,
retry_sleeps=apt.APT_RETRIES,
env={},
),
]
subp_calls += install_cmd
assert [mock.call()] == m_can_enable.call_args_list
assert 1 == m_setup_apt_proxy.call_count
assert 1 == m_installed_pkgs.call_count
assert add_apt_calls == m_add_apt.call_args_list
assert apt_pinning_calls == m_add_pinning.call_args_list
assert subp_calls == m_subp.call_args_list
assert [["", messages.FIPS_SYSTEM_REBOOT_REQUIRED.msg]
] == entitlement.cfg.read_cache("notices")
async def setUp(self):
super().setUp()
self._stack = contextlib.ExitStack().__enter__()
self._astack = await contextlib.AsyncExitStack().__aenter__()
def test_jsonarray2netcdf_execute(self):
dirname = tempfile.gettempdir()
nc_data = "Hello NetCDF!"
with contextlib.ExitStack() as stack_exec:
tmp_ncdf = tempfile.NamedTemporaryFile(dir=dirname,
mode="w",
suffix=".nc")
tmp_json = tempfile.NamedTemporaryFile(dir=dirname,
mode="w",
suffix=".json")
tmp_ncdf = stack_exec.enter_context(tmp_ncdf) # noqa
tmp_json = stack_exec.enter_context(tmp_json) # noqa
tmp_ncdf.write(nc_data)
tmp_ncdf.seek(0)
tmp_json.write(
json.dumps(
["file://{}".format(os.path.join(dirname,
tmp_ncdf.name))]))
tmp_json.seek(0)
data = {
"mode":
"async",
"response":
"document",
"inputs": [{
"id": "input",
"href": os.path.join(dirname, tmp_json.name)
}],
"outputs": [{
"id":
"output",
"transmissionMode":
EXECUTE_TRANSMISSION_MODE_REFERENCE
}],
}
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
path = "/processes/jsonarray2netcdf/jobs"
resp = mocked_sub_requests(self.app,
"post_json",
path,
data=data,
headers=self.json_headers,
only_local=True)
assert resp.status_code == 201, "Error: {}".format(resp.json)
assert resp.content_type in CONTENT_TYPE_APP_JSON
job_url = resp.json["location"]
results = self.monitor_job(job_url)
assert results["outputs"][0]["id"] == "output"
nc_path = results["outputs"][0]["href"]
assert isinstance(nc_path, str) and len(nc_path)
settings = get_settings_from_testapp(self.app)
wps_out = "{}{}".format(settings.get("weaver.url"),
settings.get("weaver.wps_output_path"))
nc_real_path = nc_path.replace(wps_out,
settings.get("weaver.wps_output_dir"))
assert nc_path.startswith(wps_out)
assert os.path.split(nc_real_path)[-1] == os.path.split(nc_path)[-1]
assert os.path.isfile(nc_real_path)
with open(nc_real_path, "r") as f:
assert f.read() == nc_data
def __init__(self):
self._serializer = cattr.Converter()
self._serializer.register_unstructure_hook(enum.Enum, _unstructure_enum)
self._serializer.register_unstructure_hook(
LoadRequest,
_with_api_key("load", self._serializer.unstructure_attrs_asdict),
)
self._serializer.register_unstructure_hook(
CreateRequest,
_with_api_key("create", self._serializer.unstructure_attrs_asdict),
)
self._serializer.register_unstructure_hook(
DeleteRequest,
_with_api_key("del", self._serializer.unstructure_attrs_asdict),
)
self._serializer.register_unstructure_hook(
GetRequest, _with_api_key("get", self._serializer.unstructure_attrs_asdict)
)
self._serializer.register_unstructure_hook(
StaticGetRequest,
_with_api_key("sget", self._serializer.unstructure_attrs_asdict),
)
self._serializer.register_unstructure_hook(
SetRequest, _with_api_key("set", self._serializer.unstructure_attrs_asdict)
)
self._serializer.register_unstructure_hook(
StaticSetRequest,
_with_api_key("sset", self._serializer.unstructure_attrs_asdict),
)
self._serializer.register_unstructure_hook(
InvokeRequest,
_with_api_key("invoke", self._serializer.unstructure_attrs_asdict),
)
self._serializer.register_unstructure_hook(
StaticInvokeRequest,
_with_api_key("sinvoke", self._serializer.unstructure_attrs_asdict),
)
self._serializer.register_unstructure_hook(
BeginRequest,
_with_api_key("begin", self._serializer.unstructure_attrs_asdict),
)
self._serializer.register_unstructure_hook(
EndRequest, _with_api_key("end", self._serializer.unstructure_attrs_asdict)
)
self._serializer.register_unstructure_hook(
CallbacksRequest,
_with_api_key("callbacks", self._serializer.unstructure_attrs_asdict),
)
self._serializer.register_unstructure_hook(
CompleteRequest,
_with_api_key("complete", self._serializer.unstructure_attrs_asdict),
)
self._serializer.register_unstructure_hook(
StatsRequest,
_with_api_key("stats", self._serializer.unstructure_attrs_asdict),
)
self._serializer.register_unstructure_hook(
Override, self._serializer.unstructure_attrs_asdict
)
self._serializer.register_unstructure_hook(ObjRef, _unstructure_ref)
self._serializer.register_structure_hook(ObjRef, _with_reference)
self._ctx_stack = contextlib.ExitStack()
def main():
parser = argparse.ArgumentParser(
description='utility for scheduling Postgresql jobs',
)
parser.add_argument(
'--check-config-only',
action='store_true',
help='do nothing. read and check config only',
)
parser.add_argument(
'--not-use-sd-notify',
action='store_true',
help='not use sd_notify',
)
parser.add_argument(
'--log-config',
metavar='LOG_CONFIG_PATH',
help='path to log config file',
)
parser.add_argument(
'config',
nargs='+',
metavar='CONFIG_PATH',
help='path to config file',
)
args = parser.parse_args()
check_config_only = args.check_config_only
not_use_sd_notify = args.not_use_sd_notify
log_config_path = args.log_config
config_path_list = args.config
with contextlib.ExitStack() as stack:
log.init(log_config_path)
config_ctx = pg_perfect_ticker.ConfigCtx()
pg_perfect_ticker.blocking_read_config(config_ctx, config_path_list)
if check_config_only:
return
loop = asyncio.get_event_loop()
ticker_ctx = pg_perfect_ticker.TickerCtx()
ticker_init_fut = loop.create_task(
pg_perfect_ticker.ticker_init(loop, ticker_ctx, config_path_list, config_ctx),
)
loop.run_until_complete(ticker_init_fut)
stack.callback(pg_perfect_ticker.blocking_ticker_shutdown, ticker_ctx)
def shutdown_handler():
loop.create_task(
pg_perfect_ticker.ticker_shutdown_handler(loop, ticker_ctx),
)
loop.add_signal_handler(signal.SIGINT, shutdown_handler)
stack.callback(loop.remove_signal_handler, signal.SIGINT)
loop.add_signal_handler(signal.SIGTERM, shutdown_handler)
stack.callback(loop.remove_signal_handler, signal.SIGTERM)
if not not_use_sd_notify:
sd.notify('READY=1', unset_environment=True)
ticker_process_fut = loop.create_task(
pg_perfect_ticker.ticker_process(loop, ticker_ctx),
)
loop.run_until_complete(ticker_process_fut)
def before_scenario(context, scenario):
"""Perform the setup before each scenario is run."""
context.resource_manager = contextlib.ExitStack()
def joy_detector(num_frames, preview_alpha, image_format, image_folder,
enable_streaming, streaming_bitrate, mdns_name):
done = threading.Event()
def stop():
logger.info('Stopping...')
done.set()
signal.signal(signal.SIGINT, lambda signum, frame: stop())
signal.signal(signal.SIGTERM, lambda signum, frame: stop())
logger.info('Starting Edit detection joy...')
with contextlib.ExitStack() as stack:
leds = stack.enter_context(Leds())
board = stack.enter_context(Board())
player = stack.enter_context(Player(gpio=BUZZER_GPIO, bpm=10))
photographer = stack.enter_context(
Photographer(image_format, image_folder))
animator = stack.enter_context(Animator(leds))
# Forced sensor mode, 1640x1232, full FoV. See:
# https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes
# This is the resolution inference run on.
# Use half of that for video streaming (820x616).
camera = stack.enter_context(
PiCamera(sensor_mode=4, resolution=(820, 616)))
stack.enter_context(PrivacyLed(leds))
server = None
if enable_streaming:
server = stack.enter_context(
StreamingServer(camera,
bitrate=streaming_bitrate,
mdns_name=mdns_name))
def model_loaded():
logger.info('Model loaded.')
player.play(MODEL_LOAD_SOUND)
def take_photo():
logger.info('Button pressed.')
player.play(BEEP_SOUND)
photographer.shoot(camera)
if preview_alpha > 0:
camera.start_preview(alpha=preview_alpha)
board.button.when_pressed = take_photo
joy_moving_average = moving_average(10)
joy_moving_average.send(None) # Initialize.
joy_threshold_detector = threshold_detector(JOY_SCORE_LOW,
JOY_SCORE_HIGH)
joy_threshold_detector.send(None) # Initialize.
for faces, frame_size in run_inference(num_frames, model_loaded):
photographer.update_faces((faces, frame_size))
joy_score = joy_moving_average.send(average_joy_score(faces))
animator.update_joy_score(joy_score)
# Send http request
sendJoy(joy_score, faces)
event = joy_threshold_detector.send(joy_score)
if event == 'high':
logger.info('High joy detected.')
player.play(JOY_SOUND)
elif event == 'low':
logger.info('Low joy detected.')
player.play(SAD_SOUND)
if server:
server.send_overlay(svg_overlay(faces, frame_size, joy_score))
if done.is_set():
break
async def _render_wfmodule(
chroot_context: ChrootContext,
workflow: Workflow,
wf_module: WfModule,
module_zipfile: Optional[ModuleZipfile],
raw_params: Dict[str, Any],
tab: Tab,
input_result: RenderResult,
tab_results: Dict[Tab, Optional[RenderResult]],
output_path: Path,
) -> RenderResult:
"""
Prepare and call `wf_module`'s `render()`; return a RenderResult.
The actual render runs in a background thread so the event loop can process
other events.
"""
basedir = output_path.parent
if wf_module.order > 0 and input_result.status != "ok":
return RenderResult() # 'unreachable'
if module_zipfile is None:
return RenderResult(errors=[
RenderError(
I18nMessage.trans(
"py.renderer.execute.wf_module.noModule",
default=
"Please delete this step: an administrator uninstalled its code.",
))
])
# exit_stack: stuff that gets deleted when the render is done
with contextlib.ExitStack() as exit_stack:
try:
# raise UnneededExecution, TabCycleError, TabOutputUnreachableError,
# PromptingError
fetch_result, params = await _execute_wfmodule_pre(
basedir,
exit_stack,
workflow,
wf_module,
module_zipfile,
raw_params,
input_result.table,
tab_results,
)
except TabCycleError:
return RenderResult(errors=[
RenderError(
I18nMessage.trans(
"py.renderer.execute.wf_module.TabCycleError",
default=
"The chosen tab depends on this one. Please choose another tab.",
))
])
except TabOutputUnreachableError:
return RenderResult(errors=[
RenderError(
I18nMessage.trans(
"py.renderer.execute.wf_module.TabOutputUnreachableError",
default=
"The chosen tab has no output. Please select another one.",
))
])
except PromptingError as err:
return RenderResult(errors=err.as_render_errors())
# Render may take a while. run_in_executor to push that slowdown to a
# thread and keep our event loop responsive.
loop = asyncio.get_event_loop()
return await loop.run_in_executor(
None,
_wrap_render_errors,
partial(
invoke_render,
module_zipfile,
chroot_context=chroot_context,
basedir=basedir,
input_table=input_result.table,
params=params,
tab=tab,
fetch_result=fetch_result,
output_filename=output_path.name,
),
)
def setUp(self):
super().setUp()
stack = contextlib.ExitStack()
self.addCleanup(stack.close)
stack.enter_context(
test_lib.ConfigOverrider({"Client.use_memory_sandboxing": True}))
def initial_sync_mirror(
channel_name: str,
remote_repository: RemoteRepository,
arch: str,
dao: Dao,
pkgstore: PackageStore,
auth: authorization.Rules,
skip_errors: bool = True,
):
force = True # needed for updating packages
try:
repo_file = remote_repository.open(os.path.join(arch, "repodata.json"))
repodata = json.load(repo_file.file)
except RemoteServerError:
logger.error(f"can not get repodata.json for channel {channel_name}")
return
except json.JSONDecodeError:
logger.error(
f"repodata.json badly formatted for arch {arch} in channel {channel_name}"
)
return
channel = dao.get_channel(channel_name)
from quetz.main import handle_package_files
packages = repodata.get("packages", {})
version_methods = [
_check_timestamp(channel, dao),
_check_checksum(pkgstore, channel_name, arch, "sha256"),
_check_checksum(pkgstore, channel_name, arch, "md5"),
]
# version_methods are context managers (for example, to update the db
# after all packages have been checked), so we need to enter the context
# for each
any_updated = False
with contextlib.ExitStack() as version_stack:
version_checks = [
version_stack.enter_context(method) for method in version_methods
]
for package_name, metadata in packages.items():
path = os.path.join(arch, package_name)
# try to find out whether it's a new package version
is_uptodate = None
for _check in version_checks:
is_uptodate = _check(package_name, metadata)
if is_uptodate is not None:
break
# if package is up-to-date skip uploading file
if is_uptodate:
logger.debug(
f"package {package_name} from {arch} up-to-date. Not updating"
)
continue
else:
logger.debug(f"updating package {package_name} form {arch}")
try:
remote_package = remote_repository.open(path)
except RemoteServerError:
logger.error(f"remote server error when getting a file {path}")
continue
files = [remote_package]
try:
handle_package_files(
channel_name,
files,
dao,
auth,
force,
)
any_updated = True
except Exception as exc:
logger.error(
f"could not process package {package_name} from channel"
f"{channel_name} due to error {exc}")
if not skip_errors:
raise exc
if any_updated:
indexing.update_indexes(dao, pkgstore, channel_name, subdirs=[arch])
